diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 398734a80db..aa9f546fa19 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - java: [21, 24] + java: [21, 25] runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution @@ -63,7 +63,7 @@ jobs: fail-fast: false matrix: os: [ windows-latest, macos-14 ] - java: [21, 24] + java: [21, 25] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index 74cf95a439a..695aa7d285d 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -20,7 +20,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [21, 24] + java: [21, 25] runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 151cf0f16d4..0e7b4c228d2 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -29,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - java: [21, 24] + java: [21, 25] test-type: ['unit', 'integration', 'doc'] runs-on: ubuntu-latest container: @@ -106,16 +106,16 @@ jobs: matrix: entry: - { os: windows-latest, java: 21, os_build_args: -PbuildPlatform=windows } - - { os: windows-latest, java: 24, os_build_args: -PbuildPlatform=windows } + - { os: windows-latest, java: 25, os_build_args: -PbuildPlatform=windows } - { os: macos-14, java: 21, os_build_args: '' } - - { os: macos-14, java: 24, os_build_args: '' } + - { os: macos-14, java: 25, os_build_args: '' } test-type: ['unit', 'integration', 'doc'] exclude: # Exclude doctest for Windows - test-type: doc entry: { os: windows-latest, java: 21, os_build_args: -PbuildPlatform=windows } - test-type: doc - entry: { os: windows-latest, java: 24, os_build_args: -PbuildPlatform=windows } + entry: { os: windows-latest, java: 25, os_build_args: -PbuildPlatform=windows } runs-on: ${{ matrix.entry.os }} steps: @@ -184,7 +184,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [21, 24] + java: [21, 25] container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -230,7 +230,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [21, 24] + java: [21, 25] container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index 096f1e5fcdc..4baf128f004 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -20,7 +20,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [21, 24] + java: [21, 25] runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution diff --git a/api/build.gradle b/api/build.gradle index 0b96acabec1..dfd0e25b902 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -28,7 +28,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index ece332519b7..5e9ce267676 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -80,7 +80,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/async-query/build.gradle b/async-query/build.gradle index a1613b5c419..417d2ab798f 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -95,12 +95,22 @@ jacocoTestCoverageVerification { rule { element = 'CLASS' excludes = [ + 'org.opensearch.sql.spark.asyncquery.OpenSearchAsyncQueryJobMetadataStorageService', 'org.opensearch.sql.spark.cluster.ClusterManagerEventListener*', 'org.opensearch.sql.spark.cluster.FlintIndexRetention', + 'org.opensearch.sql.spark.cluster.FlintStreamingJobHouseKeeperTask', 'org.opensearch.sql.spark.cluster.IndexCleanup', + 'org.opensearch.sql.spark.config.OpenSearchSparkSubmitParameterModifier', + 'org.opensearch.sql.spark.execution.session.*', + 'org.opensearch.sql.spark.execution.statement.*', // ignore because XContext IOException 'org.opensearch.sql.spark.execution.statestore.StateStore', + 'org.opensearch.sql.spark.execution.statestore.OpenSearchStatementStorageService', + 'org.opensearch.sql.spark.execution.statestore.OpenSearchSessionStorageService', + 'org.opensearch.sql.spark.flint.*', + 'org.opensearch.sql.spark.metrics.OpenSearchMetricsService', 'org.opensearch.sql.spark.rest.*', + 'org.opensearch.sql.spark.response.OpenSearchJobExecutionResponseReader', 'org.opensearch.sql.spark.scheduler.parser.OpenSearchScheduleQueryJobRequestParser', 'org.opensearch.sql.spark.transport.model.*' ] diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 05e35348784..3b59b92f943 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -5,7 +5,7 @@ plugins { id 'java-library' - id "me.champeau.jmh" version "0.6.8" + id "me.champeau.jmh" version "0.7.3" } repositories { diff --git a/build.gradle b/build.gradle index 5c4bc4ee7ba..547c2d01dd5 100644 --- a/build.gradle +++ b/build.gradle @@ -80,7 +80,7 @@ plugins { id 'java-library' id "io.freefair.lombok" version "8.14" id 'jacoco' - id 'com.diffplug.spotless' version '7.2.1' + id 'com.diffplug.spotless' version '8.1.0' } // import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94 @@ -113,7 +113,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } @@ -128,8 +128,16 @@ allprojects { } plugins.withId('java') { - sourceCompatibility = targetCompatibility = JavaVersion.VERSION_21 + java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 + } + } + + tasks.withType(Test) { + jvmArgs '-Dnet.bytebuddy.experimental=true' } + configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10" resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}" diff --git a/common/build.gradle b/common/build.gradle index ac67a003fab..bd7091819c7 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -80,6 +80,6 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/core/build.gradle b/core/build.gradle index e53749d1593..8329a5ff73c 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -26,7 +26,7 @@ plugins { id 'java-library' id "io.freefair.lombok" id 'jacoco' - id 'info.solidsoft.pitest' version '1.9.0' + id 'info.solidsoft.pitest' version '1.19.0-rc.2' id 'java-test-fixtures' id 'com.diffplug.spotless' @@ -38,7 +38,7 @@ repositories { pitest { targetClasses = ['org.opensearch.sql.*'] - pitestVersion = '1.9.0' + pitestVersion = '1.19.0-rc.2' threads = 4 outputFormats = ['HTML', 'XML'] timestampedReports = false @@ -91,7 +91,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 4848415c360..397a5f1424e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -451,8 +451,9 @@ private List expandProjectFields( .filter(addedFields::add) .forEach(field -> expandedFields.add(context.relBuilder.field(field))); } - default -> throw new IllegalStateException( - "Unexpected expression type in project list: " + expr.getClass().getSimpleName()); + default -> + throw new IllegalStateException( + "Unexpected expression type in project list: " + expr.getClass().getSimpleName()); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java index c353271d370..9b8ac7dfc97 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java +++ b/core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java @@ -171,8 +171,9 @@ public RexNode makeCast( "Cannot convert %s to IP, only STRING and IP types are supported", argExprType)); } - default -> throw new SemanticCheckException( - String.format(Locale.ROOT, "Cannot cast from %s to %s", argExprType, udt.name())); + default -> + throw new SemanticCheckException( + String.format(Locale.ROOT, "Cannot cast from %s to %s", argExprType, udt.name())); }; } // Use a custom operator when casting floating point or decimal number to a character type. diff --git a/core/src/main/java/org/opensearch/sql/calcite/type/ExprSqlType.java b/core/src/main/java/org/opensearch/sql/calcite/type/ExprSqlType.java index 8d9b90042dc..9670646e2ee 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/type/ExprSqlType.java +++ b/core/src/main/java/org/opensearch/sql/calcite/type/ExprSqlType.java @@ -54,7 +54,8 @@ public Type getJavaType() { INTEGER, INTERVAL_YEAR, INTERVAL_YEAR_MONTH, - INTERVAL_MONTH -> this.isNullable() ? Integer.class : int.class; + INTERVAL_MONTH -> + this.isNullable() ? Integer.class : int.class; case TIMESTAMP, TIMESTAMP_WITH_LOCAL_TIME_ZONE, TIMESTAMP_TZ, @@ -68,13 +69,14 @@ public Type getJavaType() { INTERVAL_HOUR_SECOND, INTERVAL_MINUTE, INTERVAL_MINUTE_SECOND, - INTERVAL_SECOND -> this.isNullable() ? Long.class : long.class; + INTERVAL_SECOND -> + this.isNullable() ? Long.class : long.class; case SMALLINT -> this.isNullable() ? Short.class : short.class; case TINYINT -> this.isNullable() ? Byte.class : byte.class; case DECIMAL -> BigDecimal.class; case BOOLEAN -> this.isNullable() ? Boolean.class : boolean.class; case DOUBLE, FLOAT -> // sic - this.isNullable() ? Double.class : double.class; + this.isNullable() ? Double.class : double.class; case REAL -> this.isNullable() ? Float.class : float.class; case BINARY, VARBINARY -> ByteString.class; case GEOMETRY -> Geometry.class; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index d225a797285..c505a431c27 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -240,7 +240,8 @@ public static ExprType convertSqlTypeNameToExprType(SqlTypeName sqlTypeName) { INTERVAL_HOUR_SECOND, INTERVAL_MINUTE, INTERVAL_MINUTE_SECOND, - INTERVAL_SECOND -> INTERVAL; + INTERVAL_SECOND -> + INTERVAL; case ARRAY -> ARRAY; case MAP -> STRUCT; case GEOMETRY -> GEO_POINT; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index c1c572225fe..cd9abcf32ba 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -156,8 +156,8 @@ static RexNode makeOver( RexWindowBound lowerBound = convert(context, windowFrame.getLower()); RexWindowBound upperBound = convert(context, windowFrame.getUpper()); switch (functionName) { - // There is no "avg" AggImplementor in Calcite, we have to change avg window - // function to `sum over(...).toRex / count over(...).toRex` + // There is no "avg" AggImplementor in Calcite, we have to change avg window + // function to `sum over(...).toRex / count over(...).toRex` case AVG: // avg(x) ==> // sum(x) / count(x) @@ -167,17 +167,17 @@ static RexNode makeOver( context.relBuilder.cast( countOver(context, field, partitions, rows, lowerBound, upperBound), SqlTypeName.DOUBLE)); - // stddev_pop(x) ==> - // power((sum(x * x) - sum(x) * sum(x) / count(x)) / count(x), 0.5) - // - // stddev_samp(x) ==> - // power((sum(x * x) - sum(x) * sum(x) / count(x)) / (count(x) - 1), 0.5) - // - // var_pop(x) ==> - // (sum(x * x) - sum(x) * sum(x) / count(x)) / count(x) - // - // var_samp(x) ==> - // (sum(x * x) - sum(x) * sum(x) / count(x)) / (count(x) - 1) + // stddev_pop(x) ==> + // power((sum(x * x) - sum(x) * sum(x) / count(x)) / count(x), 0.5) + // + // stddev_samp(x) ==> + // power((sum(x * x) - sum(x) * sum(x) / count(x)) / (count(x) - 1), 0.5) + // + // var_pop(x) ==> + // (sum(x * x) - sum(x) * sum(x) / count(x)) / count(x) + // + // var_samp(x) ==> + // (sum(x * x) - sum(x) * sum(x) / count(x)) / (count(x) - 1) case STDDEV_POP: return variance(context, field, partitions, rows, lowerBound, upperBound, true, true); case STDDEV_SAMP: diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 32aee242388..f619d966cc8 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -131,8 +131,8 @@ public static SqlTypeName convertRelDataTypeToSqlTypeName(RelDataType type) { case EXPR_DATE -> SqlTypeName.DATE; case EXPR_TIME -> SqlTypeName.TIME; case EXPR_TIMESTAMP -> SqlTypeName.TIMESTAMP; - // EXPR_IP is mapped to SqlTypeName.OTHER since there is no - // corresponding SqlTypeName in Calcite. + // EXPR_IP is mapped to SqlTypeName.OTHER since there is no + // corresponding SqlTypeName in Calcite. case EXPR_IP -> SqlTypeName.OTHER; case EXPR_BINARY -> SqlTypeName.VARBINARY; default -> type.getSqlTypeName(); diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/time/TimeSpanExpressionFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/time/TimeSpanExpressionFactory.java index 62ccb807801..a205de2f4c9 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/binning/time/TimeSpanExpressionFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/binning/time/TimeSpanExpressionFactory.java @@ -34,14 +34,9 @@ public RexNode createTimeSpanExpression( TimeUnitRegistry.validateSubSecondSpan(config, intervalValue); return switch (config) { - case MICROSECONDS, - MILLISECONDS, - CENTISECONDS, - DECISECONDS, - SECONDS, - MINUTES, - HOURS -> standardHandler.createExpression( - fieldExpr, intervalValue, config, alignmentOffsetMillis, context); + case MICROSECONDS, MILLISECONDS, CENTISECONDS, DECISECONDS, SECONDS, MINUTES, HOURS -> + standardHandler.createExpression( + fieldExpr, intervalValue, config, alignmentOffsetMillis, context); case DAYS -> dayHandler.createExpression(fieldExpr, intervalValue, context); case MONTHS -> monthHandler.createExpression(fieldExpr, intervalValue, context); }; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/datetime/DateTimeConversionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/datetime/DateTimeConversionUtils.java index ebc4f56c748..9181ca0eb76 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/datetime/DateTimeConversionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/datetime/DateTimeConversionUtils.java @@ -30,16 +30,18 @@ public static ExprTimestampValue forceConvertToTimestampValue( ExprValue value, FunctionProperties properties) { return switch (value) { case ExprTimestampValue timestampValue -> timestampValue; - case ExprDateValue dateValue -> (ExprTimestampValue) - ExprValueUtils.timestampValue(dateValue.timestampValue()); - case ExprTimeValue timeValue -> (ExprTimestampValue) - ExprValueUtils.timestampValue(timeValue.timestampValue(properties)); - case ExprStringValue stringValue -> new ExprTimestampValue( - DateTimeParser.parse(stringValue.stringValue())); - default -> throw new ExpressionEvaluationException( - String.format( - "Cannot convert %s to timestamp, only STRING, DATE, TIME and TIMESTAMP are supported", - value.type())); + case ExprDateValue dateValue -> + (ExprTimestampValue) ExprValueUtils.timestampValue(dateValue.timestampValue()); + case ExprTimeValue timeValue -> + (ExprTimestampValue) ExprValueUtils.timestampValue(timeValue.timestampValue(properties)); + case ExprStringValue stringValue -> + new ExprTimestampValue(DateTimeParser.parse(stringValue.stringValue())); + default -> + throw new ExpressionEvaluationException( + String.format( + "Cannot convert %s to timestamp, only STRING, DATE, TIME and TIMESTAMP are" + + " supported", + value.type())); }; } @@ -129,8 +131,9 @@ public static TemporalAmount convertToTemporalAmount(long number, TimeUnit unit) case MICROSECOND -> Duration.ofNanos(number * 1000); case NANOSECOND -> Duration.ofNanos(number); - default -> throw new UnsupportedOperationException( - "No mapping defined for Calcite TimeUnit: " + unit); + default -> + throw new UnsupportedOperationException( + "No mapping defined for Calcite TimeUnit: " + unit); }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java index f5d3c44b7e1..e0f626e54f1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunctions.java @@ -2060,23 +2060,23 @@ private DateTimeFormatter getFormatter(int dateAsInt) { // Check below from YYYYMMDD - MMDD which format should be used switch (length) { - // Check if dateAsInt is at least 8 digits long + // Check if dateAsInt is at least 8 digits long case FULL_DATE_LENGTH: return DATE_FORMATTER_LONG_YEAR; - // Check if dateAsInt is at least 6 digits long + // Check if dateAsInt is at least 6 digits long case SHORT_DATE_LENGTH: return DATE_FORMATTER_SHORT_YEAR; - // Check if dateAsInt is at least 5 digits long + // Check if dateAsInt is at least 5 digits long case SINGLE_DIGIT_YEAR_DATE_LENGTH: return DATE_FORMATTER_SINGLE_DIGIT_YEAR; - // Check if dateAsInt is at least 4 digits long + // Check if dateAsInt is at least 4 digits long case NO_YEAR_DATE_LENGTH: return DATE_FORMATTER_NO_YEAR; - // Check if dateAsInt is at least 3 digits long + // Check if dateAsInt is at least 3 digits long case SINGLE_DIGIT_MONTH_DATE_LENGTH: return DATE_FORMATTER_SINGLE_DIGIT_MONTH; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java index bb58a38a109..521764ba7bb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLTypeChecker.java @@ -600,25 +600,28 @@ private static List> getExprSignatures(FamilyOperandTypeChecker t private static List getExprTypes(SqlTypeFamily family) { List concreteTypes = switch (family) { - case DATETIME -> List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DATE), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIME)); - case NUMERIC -> List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER), - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE)); - // Integer is mapped to BIGINT in family.getDefaultConcreteType - case INTEGER -> List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER)); - case ANY, IGNORE -> List.of( - OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY)); - case DATETIME_INTERVAL -> SqlTypeName.INTERVAL_TYPES.stream() - .map( - type -> - OpenSearchTypeFactory.TYPE_FACTORY.createSqlIntervalType( - new SqlIntervalQualifier( - type.getStartUnit(), type.getEndUnit(), SqlParserPos.ZERO))) - .collect(Collectors.toList()); + case DATETIME -> + List.of( + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DATE), + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIME)); + case NUMERIC -> + List.of( + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER), + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.DOUBLE)); + // Integer is mapped to BIGINT in family.getDefaultConcreteType + case INTEGER -> + List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER)); + case ANY, IGNORE -> + List.of(OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.ANY)); + case DATETIME_INTERVAL -> + SqlTypeName.INTERVAL_TYPES.stream() + .map( + type -> + OpenSearchTypeFactory.TYPE_FACTORY.createSqlIntervalType( + new SqlIntervalQualifier( + type.getStartUnit(), type.getEndUnit(), SqlParserPos.ZERO))) + .collect(Collectors.toList()); default -> { RelDataType type = family.getDefaultConcreteType(OpenSearchTypeFactory.TYPE_FACTORY); if (type == null) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java index bb228a1b0e2..0f46e5cbb1c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java @@ -50,15 +50,17 @@ public Expression implement( public static String getDigest(String input, int algorithm) { return switch (algorithm) { - case 224 -> Hex.encodeHexString( - DigestUtils.getDigest(MessageDigestAlgorithms.SHA_224).digest(input.getBytes())); + case 224 -> + Hex.encodeHexString( + DigestUtils.getDigest(MessageDigestAlgorithms.SHA_224).digest(input.getBytes())); case 256 -> DigestUtils.sha256Hex(input); case 384 -> DigestUtils.sha384Hex(input); case 512 -> DigestUtils.sha512Hex(input); - default -> throw new IllegalArgumentException( - String.format( - "Unsupported SHA2 algorithm: %d. Only 224, 256, 384, and 512 are supported.", - algorithm)); + default -> + throw new IllegalArgumentException( + String.format( + "Unsupported SHA2 algorithm: %d. Only 224, 256, 384, and 512 are supported.", + algorithm)); }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java index ed32872c8e2..f28f12e30b9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java @@ -88,11 +88,12 @@ public Expression implement( } if (SqlTypeUtil.isNull(unitType)) { return switch (call.getType().getSqlTypeName()) { - case BIGINT, INTEGER, SMALLINT, TINYINT -> Expressions.multiply( - Expressions.divide(field, interval), interval); - default -> Expressions.multiply( - Expressions.call(BuiltInMethod.FLOOR.method, Expressions.divide(field, interval)), - interval); + case BIGINT, INTEGER, SMALLINT, TINYINT -> + Expressions.multiply(Expressions.divide(field, interval), interval); + default -> + Expressions.multiply( + Expressions.call(BuiltInMethod.FLOOR.method, Expressions.divide(field, interval)), + interval); }; } else if (fieldType instanceof ExprSqlType exprSqlType) { // TODO: pass in constant arguments when constructing @@ -101,8 +102,9 @@ public Expression implement( case EXPR_DATE -> "evalDate"; case EXPR_TIME -> "evalTime"; case EXPR_TIMESTAMP -> "evalTimestamp"; - default -> throw new IllegalArgumentException( - String.format("Unsupported expr type: %s", exprSqlType.getExprType())); + default -> + throw new IllegalArgumentException( + String.format("Unsupported expr type: %s", exprSqlType.getExprType())); }; ScalarFunctionImpl function = (ScalarFunctionImpl) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java index 719f078ba38..c6ff1a64478 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/condition/EnhancedCoalesceFunction.java @@ -63,18 +63,18 @@ private static ExprValue coerceToType(ExprValue value, String typeName) { return switch (typeName) { case "INTEGER" -> tryConvert(() -> ExprValueUtils.integerValue(value.integerValue()), value); case "BIGINT" -> tryConvert(() -> ExprValueUtils.longValue(value.longValue()), value); - case "SMALLINT", "TINYINT" -> tryConvert( - () -> ExprValueUtils.integerValue(value.integerValue()), value); + case "SMALLINT", "TINYINT" -> + tryConvert(() -> ExprValueUtils.integerValue(value.integerValue()), value); case "DOUBLE" -> tryConvert(() -> ExprValueUtils.doubleValue(value.doubleValue()), value); - case "FLOAT", "REAL" -> tryConvert( - () -> ExprValueUtils.floatValue(value.floatValue()), value); + case "FLOAT", "REAL" -> + tryConvert(() -> ExprValueUtils.floatValue(value.floatValue()), value); case "BOOLEAN" -> tryConvert(() -> ExprValueUtils.booleanValue(value.booleanValue()), value); - case "VARCHAR", "CHAR" -> tryConvert( - () -> ExprValueUtils.stringValue(String.valueOf(value.value())), value); + case "VARCHAR", "CHAR" -> + tryConvert(() -> ExprValueUtils.stringValue(String.valueOf(value.value())), value); case "DATE" -> tryConvert(() -> ExprValueUtils.dateValue(value.dateValue()), value); case "TIME" -> tryConvert(() -> ExprValueUtils.timeValue(value.timeValue()), value); - case "TIMESTAMP" -> tryConvert( - () -> ExprValueUtils.timestampValue(value.timestampValue()), value); + case "TIMESTAMP" -> + tryConvert(() -> ExprValueUtils.timestampValue(value.timestampValue()), value); case "DECIMAL" -> tryConvert(() -> ExprValueUtils.doubleValue(value.doubleValue()), value); default -> value; }; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index 96d59a0a704..ce200323f60 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -99,9 +99,10 @@ public boolean isDeterministic() { case GREATER_THAN_OR_EQUAL -> PPLBuiltinOperators.LTE_IP; case EQUALS -> PPLBuiltinOperators.EQUALS_IP; case NOT_EQUALS -> PPLBuiltinOperators.NOT_EQUALS_IP; - default -> throw new IllegalArgumentException( - String.format( - Locale.ROOT, "CompareIpFunction is not supposed to be of kind: %s", kind)); + default -> + throw new IllegalArgumentException( + String.format( + Locale.ROOT, "CompareIpFunction is not supposed to be of kind: %s", kind)); }; } @@ -151,8 +152,9 @@ private static Expression evalCompareResult(Expression compareResult, SqlKind co case LESS_THAN_OR_EQUAL -> Expressions.lessThanOrEqual(compareResult, zero); case GREATER_THAN -> Expressions.greaterThan(compareResult, zero); case GREATER_THAN_OR_EQUAL -> Expressions.greaterThanOrEqual(compareResult, zero); - default -> throw new UnsupportedOperationException( - String.format(Locale.ROOT, "Unsupported compare type: %s", compareType)); + default -> + throw new UnsupportedOperationException( + String.format(Locale.ROOT, "Unsupported compare type: %s", compareType)); }; } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java index ab97f3f4df1..8c5661021f5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java @@ -251,7 +251,7 @@ private Temporal extractValue(FunctionExpression func) { return LocalDateTime.ofInstant(func.valueOf().timestampValue(), ZoneOffset.UTC); case TIME: return func.valueOf().timeValue(); - // unreachable code + // unreachable code default: throw new IllegalArgumentException(String.format("%s", func.type())); } diff --git a/datasources/build.gradle b/datasources/build.gradle index cdea790bf97..1dd01d82fb9 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -35,6 +35,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } test { diff --git a/direct-query-core/build.gradle b/direct-query-core/build.gradle index 25b0f7869c1..7caffd11ffa 100644 --- a/direct-query-core/build.gradle +++ b/direct-query-core/build.gradle @@ -41,6 +41,7 @@ dependencies { exclude group: 'org.hamcrest', module: 'hamcrest-core' } testImplementation("org.opensearch.test:framework:${opensearch_version}") + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } test { diff --git a/direct-query/build.gradle b/direct-query/build.gradle index e2b70df77b3..2a1dc04f6a1 100644 --- a/direct-query/build.gradle +++ b/direct-query/build.gradle @@ -43,6 +43,7 @@ dependencies { exclude group: 'org.hamcrest', module: 'hamcrest-core' } testImplementation("org.opensearch.test:framework:${opensearch_version}") + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } test { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index a4b76b9530d..1b33c55baab 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f373f37ad82..b11741a1ada 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=efe9a3d147d948d7528a9887fa35abcf24ca1a43ad06439996490f77569b02d1 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip +distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f5feea6d6b1..23d15a93670 100755 --- a/gradlew +++ b/gradlew @@ -86,8 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s -' "$PWD" ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -115,7 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -206,7 +205,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -214,7 +213,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 9b42019c791..5eed7ee8452 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,11 @@ goto fail :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar +set CLASSPATH= @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 6df97e244a8..ea098d51b5a 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -749,7 +749,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java index 25b1368fcdd..4664491b686 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java @@ -113,24 +113,24 @@ private void createRole(String roleName, String indexPattern) throws IOException String.format( Locale.ROOT, """ - { - "cluster_permissions": [ - "cluster:admin/opensearch/ppl" - ], - "index_permissions": [{ - "index_patterns": [ - "%s" - ], - "allowed_actions": [ - "indices:data/read/search*", - "indices:admin/mappings/get", - "indices:monitor/settings/get", - "indices:data/read/point_in_time/create", - "indices:data/read/point_in_time/delete" - ] - }] - } - """, + { + "cluster_permissions": [ + "cluster:admin/opensearch/ppl" + ], + "index_permissions": [{ + "index_patterns": [ + "%s" + ], + "allowed_actions": [ + "indices:data/read/search*", + "indices:admin/mappings/get", + "indices:monitor/settings/get", + "indices:data/read/point_in_time/create", + "indices:data/read/point_in_time/delete" + ] + }] + } + """, indexPattern)); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); @@ -151,12 +151,12 @@ private void createUser(String username, String roleName) throws IOException { String.format( Locale.ROOT, """ - { - "password": "%s", - "backend_roles": [], - "attributes": {} - } - """, + { + "password": "%s", + "backend_roles": [], + "attributes": {} + } + """, STRONG_PASSWORD)); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); @@ -175,12 +175,12 @@ private void createUser(String username, String roleName) throws IOException { String.format( Locale.ROOT, """ - { - "backend_roles": [], - "hosts": [], - "users": ["%s"] - } - """, + { + "backend_roles": [], + "hosts": [], + "users": ["%s"] + } + """, username)); mappingRequest.setOptions(restOptionsBuilder); @@ -270,14 +270,14 @@ private void createRoleWithSpecificPermissions( String.format( Locale.ROOT, """ - { - "cluster_permissions": [%s], - "index_permissions": [{ - "index_patterns": ["%s"], - "allowed_actions": [%s] - }] - } - """, + { + "cluster_permissions": [%s], + "index_permissions": [{ + "index_patterns": ["%s"], + "allowed_actions": [%s] + }] + } + """, clusterPermsJson, indexPattern, indexPermsJson)); diff --git a/legacy/build.gradle b/legacy/build.gradle index 7e78d5e6e13..fd6d7c8f65c 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -68,7 +68,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java index a6a91995333..7c2e69d7829 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/utils/SQLFunctions.java @@ -159,8 +159,8 @@ public Tuple function( break; } - // Split is currently not supported since its using .split() in painless which is not - // allow-listed + // Split is currently not supported since its using .split() in painless which is not + // allow-listed case "split": if (paramers.size() == 3) { functionStr = diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 27aa81b0b67..5ca10c7091c 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -26,7 +26,7 @@ plugins { id 'java-library' id "io.freefair.lombok" id 'jacoco' - id 'info.solidsoft.pitest' version '1.9.0' + id 'info.solidsoft.pitest' version '1.19.0-rc.2' id 'com.diffplug.spotless' } @@ -47,8 +47,8 @@ dependencies { testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.3') testImplementation('org.junit.jupiter:junit-jupiter-params:5.9.3') - testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine:5.9.3') - testRuntimeOnly('org.junit.platform:junit-platform-launcher:1.9.3') + testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine') + testRuntimeOnly('org.junit.platform:junit-platform-launcher') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" @@ -72,13 +72,13 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } pitest { targetClasses = ['org.opensearch.sql.*'] - pitestVersion = '1.9.0' + pitestVersion = '1.19.0-rc.2' threads = 4 outputFormats = ['HTML', 'XML'] timestampedReports = false diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 2548c67cffe..837a2a062ef 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -144,7 +144,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType)); switch (mappingType) { case Object: - // TODO: use Object type once it has been added + // TODO: use Object type once it has been added case Nested: if (innerMap.isEmpty()) { return res; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index b4bf48bd880..2abfb5a401b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -366,13 +366,15 @@ private static Pair createDistinctAggregation( AggregateBuilderHelper helper) { return switch (aggCall.getAggregation().kind) { - case COUNT -> Pair.of( - helper.build( - !args.isEmpty() ? args.getFirst() : null, - AggregationBuilders.cardinality(aggFieldName)), - new SingleValueParser(aggFieldName)); - default -> throw new AggregateAnalyzer.AggregateAnalyzerException( - String.format("unsupported distinct aggregator %s", aggCall.getAggregation())); + case COUNT -> + Pair.of( + helper.build( + !args.isEmpty() ? args.getFirst() : null, + AggregationBuilders.cardinality(aggFieldName)), + new SingleValueParser(aggFieldName)); + default -> + throw new AggregateAnalyzer.AggregateAnalyzerException( + String.format("unsupported distinct aggregator %s", aggCall.getAggregation())); }; } @@ -383,18 +385,22 @@ private static Pair createRegularAggregation( AggregateBuilderHelper helper) { return switch (aggCall.getAggregation().kind) { - case AVG -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.avg(aggFieldName)), - new SingleValueParser(aggFieldName)); - // 1. Only case SUM, skip SUM0 / COUNT since calling avg() in DSL should be faster. - // 2. To align with databases, SUM0 is not preferred now. - case SUM -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.sum(aggFieldName)), - new SingleValueParser(aggFieldName)); - case COUNT -> Pair.of( - helper.build( - !args.isEmpty() ? args.getFirst() : null, AggregationBuilders.count(aggFieldName)), - new SingleValueParser(aggFieldName)); + case AVG -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.avg(aggFieldName)), + new SingleValueParser(aggFieldName)); + // 1. Only case SUM, skip SUM0 / COUNT since calling avg() in DSL should be faster. + // 2. To align with databases, SUM0 is not preferred now. + case SUM -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.sum(aggFieldName)), + new SingleValueParser(aggFieldName)); + case COUNT -> + Pair.of( + helper.build( + !args.isEmpty() ? args.getFirst() : null, + AggregationBuilders.count(aggFieldName)), + new SingleValueParser(aggFieldName)); case MIN -> { ExprType fieldType = OpenSearchTypeFactory.convertRelDataTypeToExprType(args.getFirst().getType()); @@ -433,46 +439,54 @@ private static Pair createRegularAggregation( new TopHitsParser(aggFieldName, true)); } } - case VAR_SAMP -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), - new StatsParser(ExtendedStats::getVarianceSampling, aggFieldName)); - case VAR_POP -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), - new StatsParser(ExtendedStats::getVariancePopulation, aggFieldName)); - case STDDEV_SAMP -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), - new StatsParser(ExtendedStats::getStdDeviationSampling, aggFieldName)); - case STDDEV_POP -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), - new StatsParser(ExtendedStats::getStdDeviationPopulation, aggFieldName)); - case ARG_MAX -> Pair.of( - AggregationBuilders.topHits(aggFieldName) - .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) - .size(1) - .from(0) - .sort( - helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), - org.opensearch.search.sort.SortOrder.DESC), - new ArgMaxMinParser(aggFieldName)); - case ARG_MIN -> Pair.of( - AggregationBuilders.topHits(aggFieldName) - .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) - .size(1) - .from(0) - .sort( - helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), - org.opensearch.search.sort.SortOrder.ASC), - new ArgMaxMinParser(aggFieldName)); + case VAR_SAMP -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getVarianceSampling, aggFieldName)); + case VAR_POP -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getVariancePopulation, aggFieldName)); + case STDDEV_SAMP -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getStdDeviationSampling, aggFieldName)); + case STDDEV_POP -> + Pair.of( + helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), + new StatsParser(ExtendedStats::getStdDeviationPopulation, aggFieldName)); + case ARG_MAX -> + Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) + .size(1) + .from(0) + .sort( + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), + org.opensearch.search.sort.SortOrder.DESC), + new ArgMaxMinParser(aggFieldName)); + case ARG_MIN -> + Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) + .size(1) + .from(0) + .sort( + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), + org.opensearch.search.sort.SortOrder.ASC), + new ArgMaxMinParser(aggFieldName)); case OTHER_FUNCTION -> { BuiltinFunctionName functionName = BuiltinFunctionName.ofAggregation(aggCall.getAggregation().getName()).get(); yield switch (functionName) { - case TAKE -> Pair.of( - AggregationBuilders.topHits(aggFieldName) - .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) - .size(helper.inferValue(args.getLast(), Integer.class)) - .from(0), - new TopHitsParser(aggFieldName)); + case TAKE -> + Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchField( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) + .size(helper.inferValue(args.getLast(), Integer.class)) + .from(0), + new TopHitsParser(aggFieldName)); case FIRST -> { TopHitsAggregationBuilder firstBuilder = AggregationBuilders.topHits(aggFieldName).size(1).from(0); @@ -505,17 +519,20 @@ yield switch (functionName) { } yield Pair.of(aggBuilder, new SinglePercentileParser(aggFieldName)); } - case DISTINCT_COUNT_APPROX -> Pair.of( - helper.build( - !args.isEmpty() ? args.getFirst() : null, - AggregationBuilders.cardinality(aggFieldName)), - new SingleValueParser(aggFieldName)); - default -> throw new AggregateAnalyzer.AggregateAnalyzerException( - String.format("Unsupported push-down aggregator %s", aggCall.getAggregation())); + case DISTINCT_COUNT_APPROX -> + Pair.of( + helper.build( + !args.isEmpty() ? args.getFirst() : null, + AggregationBuilders.cardinality(aggFieldName)), + new SingleValueParser(aggFieldName)); + default -> + throw new AggregateAnalyzer.AggregateAnalyzerException( + String.format("Unsupported push-down aggregator %s", aggCall.getAggregation())); }; } - default -> throw new AggregateAnalyzer.AggregateAnalyzerException( - String.format("unsupported aggregator %s", aggCall.getAggregation())); + default -> + throw new AggregateAnalyzer.AggregateAnalyzerException( + String.format("unsupported aggregator %s", aggCall.getAggregation())); }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java index 104ab04e547..5ec21888a89 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzer.java @@ -174,8 +174,9 @@ private void analyzeSimpleComparison(RexCall call, String key) { case LESS_THAN -> { addTo(key, value); } - default -> throw new UnsupportedOperationException( - "ranges must be equivalents of field >= constant or field < constant"); + default -> + throw new UnsupportedOperationException( + "ranges must be equivalents of field >= constant or field < constant"); } ; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 7abb0c1b937..c74356ce977 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -374,7 +374,7 @@ public Expression visitCall(RexCall call) { || MULTI_FIELDS_RELEVANCE_FUNCTION_SET.contains(functionName)) { return visitRelevanceFunc(call); } - // fall through + // fall through default: String message = format(Locale.ROOT, "Unsupported syntax [%s] for call: [%s]", syntax, call); @@ -657,14 +657,16 @@ private QueryExpression binary(RexCall call) { RexUnknownAs nullAs = getNullAsForSearch(call); QueryExpression finalExpression = switch (nullAs) { - // e.g. where isNotNull(a) and (a = 1 or a = 2) - // TODO: For this case, seems return `expression` should be equivalent - case FALSE -> CompoundQueryExpression.and( - false, expression, QueryExpression.create(pair.getKey()).exists()); - // e.g. where isNull(a) or a = 1 or a = 2 - case TRUE -> CompoundQueryExpression.or( - expression, QueryExpression.create(pair.getKey()).notExists()); - // e.g. where a = 1 or a = 2 + // e.g. where isNotNull(a) and (a = 1 or a = 2) + // TODO: For this case, seems return `expression` should be equivalent + case FALSE -> + CompoundQueryExpression.and( + false, expression, QueryExpression.create(pair.getKey()).exists()); + // e.g. where isNull(a) or a = 1 or a = 2 + case TRUE -> + CompoundQueryExpression.or( + expression, QueryExpression.create(pair.getKey()).notExists()); + // e.g. where a = 1 or a = 2 case UNKNOWN -> expression; }; finalExpression.updateAnalyzedNodes(call); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 88f15607a7a..11421fca0a1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -122,13 +122,15 @@ public double estimateRowCount(RelMetadataQuery mq) { switch (operation.type()) { case AGGREGATION -> mq.getRowCount((RelNode) operation.digest()); case PROJECT, SORT, SORT_EXPR -> rowCount; - case SORT_AGG_METRICS -> NumberUtil.min( - rowCount, osIndex.getBucketSize().doubleValue()); - // Refer the org.apache.calcite.rel.metadata.RelMdRowCount + case SORT_AGG_METRICS -> + NumberUtil.min(rowCount, osIndex.getBucketSize().doubleValue()); + // Refer the org.apache.calcite.rel.metadata.RelMdRowCount case COLLAPSE -> rowCount / 10; - case FILTER, SCRIPT -> NumberUtil.multiply( - rowCount, - RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); + case FILTER, SCRIPT -> + NumberUtil.multiply( + rowCount, + RelMdUtil.guessSelectivity( + ((FilterDigest) operation.digest()).condition())); case LIMIT -> Math.min(rowCount, ((LimitDigest) operation.digest()).limit()); case RARE_TOP -> { /** similar to {@link Aggregate#estimateRowCount(RelMetadataQuery)} */ @@ -166,7 +168,7 @@ public double estimateRowCount(RelMetadataQuery mq) { dRows = mq.getRowCount((RelNode) operation.digest()); dCpu += dRows * getAggMultiplier(operation); } - // Ignored Project in cost accumulation, but it will affect the external cost + // Ignored Project in cost accumulation, but it will affect the external cost case PROJECT -> {} case SORT -> dCpu += dRows; case SORT_AGG_METRICS -> { @@ -180,15 +182,17 @@ public double estimateRowCount(RelMetadataQuery mq) { sortKeys.stream().filter(digest -> digest.getExpression() != null).count(); dCpu += NumberUtil.multiply(dRows, 1.1 * complexExprCount); } - // Refer the org.apache.calcite.rel.metadata.RelMdRowCount.getRowCount(Aggregate rel,...) + // Refer the org.apache.calcite.rel.metadata.RelMdRowCount.getRowCount(Aggregate rel,...) case COLLAPSE -> { dRows = dRows / 10; dCpu += dRows; } - // Ignore cost the primitive filter but it will affect the rows count. - case FILTER -> dRows = - NumberUtil.multiply( - dRows, RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); + // Ignore cost the primitive filter but it will affect the rows count. + case FILTER -> + dRows = + NumberUtil.multiply( + dRows, + RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); case SCRIPT -> { FilterDigest filterDigest = (FilterDigest) operation.digest(); dRows = NumberUtil.multiply(dRows, RelMdUtil.guessSelectivity(filterDigest.condition())); @@ -196,10 +200,10 @@ public double estimateRowCount(RelMetadataQuery mq) { // the factor amplified by script count. dCpu += NumberUtil.multiply(dRows, Math.pow(1.1, filterDigest.scriptCount())); } - // Ignore cost the LIMIT but it will affect the rows count. - // Try to reduce the rows count by 1 to make the cost cheaper slightly than non-push down. - // Because we'd like to push down LIMIT even when the fetch in LIMIT is greater than - // dRows. + // Ignore cost the LIMIT but it will affect the rows count. + // Try to reduce the rows count by 1 to make the cost cheaper slightly than non-push down. + // Because we'd like to push down LIMIT even when the fetch in LIMIT is greater than + // dRows. case LIMIT -> dRows = Math.min(dRows, ((LimitDigest) operation.digest()).limit()) - 1; case RARE_TOP -> { /** similar to {@link Aggregate#computeSelfCost(RelOptPlanner, RelMetadataQuery)} */ diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java index 83ec6093718..10334bc1b26 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java @@ -61,12 +61,11 @@ public Object execute() { // See logic in {@link ExpressionAggregationScript::execute} return switch ((ExprCoreType) exprType) { case TIME -> - // Can't get timestamp from `ExprTimeValue` - MILLIS.between(LocalTime.MIN, ExprValueUtils.fromObjectValue(value, TIME).timeValue()); + // Can't get timestamp from `ExprTimeValue` + MILLIS.between(LocalTime.MIN, ExprValueUtils.fromObjectValue(value, TIME).timeValue()); case DATE -> ExprValueUtils.fromObjectValue(value, DATE).timestampValue().toEpochMilli(); - case TIMESTAMP -> ExprValueUtils.fromObjectValue(value, TIMESTAMP) - .timestampValue() - .toEpochMilli(); + case TIMESTAMP -> + ExprValueUtils.fromObjectValue(value, TIMESTAMP).timestampValue().toEpochMilli(); default -> value; }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index 83e8b8b74b1..dab778923b6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -97,7 +97,7 @@ public static Optional> getOrderEquivalentInputInfo(RexNo return getOrderEquivalentInputInfo(variable) .map(inputInfo -> Pair.of(inputInfo.getLeft(), flipped != inputInfo.getRight())); } - // Ignore DIVIDE operator for now because it has too many precision issues + // Ignore DIVIDE operator for now because it has too many precision issues case CAST, SAFE_CAST: { RexNode child = ((RexCall) expr).getOperands().get(0); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java index 505db011f7b..6205cae42ab 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/CaseRangeAnalyzerTest.java @@ -113,7 +113,8 @@ void testAnalyzeSimpleCaseExpression() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -159,7 +160,8 @@ void testAnalyzeLessThanComparison() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -216,7 +218,8 @@ void testAnalyzeWithSearchCondition() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -265,7 +268,8 @@ void testAnalyzeWithNullElse() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -444,7 +448,8 @@ void testAnalyzeWithReversedComparison() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -514,7 +519,8 @@ void testSimpleCaseGeneratesExpectedDSL() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -576,7 +582,8 @@ void testMultipleConditionsGenerateExpectedDSL() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -622,7 +629,8 @@ void testLessThanConditionGeneratesExpectedDSL() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -670,7 +678,8 @@ void testNullElseClauseGeneratesExpectedDSL() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -727,7 +736,8 @@ void testSearchConditionGeneratesExpectedDSL() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } @@ -798,7 +808,8 @@ void testSearchWithDiscontinuousRanges() { "keyed" : true } } - }"""; + }\ + """; assertEquals(normalizeJson(expectedJson), normalizeJson(builder.toString())); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 9d7a6b93f92..9486f5fe0b8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -93,14 +93,15 @@ void equals_generatesTermQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(TermQueryBuilder.class, result); assertEquals( """ - { - "term" : { - "a" : { - "value" : 12, - "boost" : 1.0 - } - } - }""", + { + "term" : { + "a" : { + "value" : 12, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -112,30 +113,31 @@ void notEquals_generatesBoolQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(BoolQueryBuilder.class, result); assertEquals( """ - { - "bool" : { - "must" : [ - { - "exists" : { - "field" : "a", - "boost" : 1.0 - } - } - ], - "must_not" : [ - { - "term" : { - "a" : { - "value" : 12, - "boost" : 1.0 - } - } + { + "bool" : { + "must" : [ + { + "exists" : { + "field" : "a", + "boost" : 1.0 + } + } + ], + "must_not" : [ + { + "term" : { + "a" : { + "value" : 12, + "boost" : 1.0 } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + } } - }""", + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -147,17 +149,18 @@ void gt_generatesRangeQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "a" : { - "from" : 12, - "to" : null, - "include_lower" : false, - "include_upper" : true, - "boost" : 1.0 - } - } - }""", + { + "range" : { + "a" : { + "from" : 12, + "to" : null, + "include_lower" : false, + "include_upper" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -170,17 +173,18 @@ void gte_generatesRangeQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "a" : { - "from" : 12, - "to" : null, - "include_lower" : true, - "include_upper" : true, - "boost" : 1.0 - } - } - }""", + { + "range" : { + "a" : { + "from" : 12, + "to" : null, + "include_lower" : true, + "include_upper" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -192,17 +196,18 @@ void lt_generatesRangeQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "a" : { - "from" : null, - "to" : 12, - "include_lower" : true, - "include_upper" : false, - "boost" : 1.0 - } - } - }""", + { + "range" : { + "a" : { + "from" : null, + "to" : 12, + "include_lower" : true, + "include_upper" : false, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -214,17 +219,18 @@ void lte_generatesRangeQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "a" : { - "from" : null, - "to" : 12, - "include_lower" : true, - "include_upper" : true, - "boost" : 1.0 - } - } - }""", + { + "range" : { + "a" : { + "from" : null, + "to" : 12, + "include_lower" : true, + "include_upper" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -236,12 +242,13 @@ void exists_generatesExistsQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(ExistsQueryBuilder.class, result); assertEquals( """ - { - "exists" : { - "field" : "a", - "boost" : 1.0 - } - }""", + { + "exists" : { + "field" : "a", + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -266,7 +273,8 @@ void notExists_generatesMustNotExistsQuery() throws ExpressionNotAnalyzableExcep "adjust_pure_negative" : true, "boost" : 1.0 } - }""", + }\ + """, result.toString()); } @@ -281,16 +289,17 @@ void search_generatesTermsQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(TermsQueryBuilder.class, result); assertEquals( """ - { - "terms" : { - "a" : [ - 12.0, - 13.0, - 14.0 - ], - "boost" : 1.0 - } - }""", + { + "terms" : { + "a" : [ + 12.0, + 13.0, + 14.0 + ], + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -302,21 +311,22 @@ void contains_generatesMatchQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(MatchQueryBuilder.class, result); assertEquals( """ - { - "match" : { - "b" : { - "query" : "Hi", - "operator" : "OR", - "prefix_length" : 0, - "max_expansions" : 50, - "fuzzy_transpositions" : true, - "lenient" : false, - "zero_terms_query" : "NONE", - "auto_generate_synonyms_phrase_query" : true, - "boost" : 1.0 - } - } - }""", + { + "match" : { + "b" : { + "query" : "Hi", + "operator" : "OR", + "prefix_length" : 0, + "max_expansions" : 50, + "fuzzy_transpositions" : true, + "lenient" : false, + "zero_terms_query" : "NONE", + "auto_generate_synonyms_phrase_query" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -329,21 +339,22 @@ void matchRelevanceQueryFunction_generatesMatchQuery() throws ExpressionNotAnaly assertInstanceOf(MatchQueryBuilder.class, result); assertEquals( """ - { - "match" : { - "b" : { - "query" : "Hi", - "operator" : "OR", - "prefix_length" : 0, - "max_expansions" : 50, - "fuzzy_transpositions" : true, - "lenient" : false, - "zero_terms_query" : "NONE", - "auto_generate_synonyms_phrase_query" : true, - "boost" : 1.0 - } - } - }""", + { + "match" : { + "b" : { + "query" : "Hi", + "operator" : "OR", + "prefix_length" : 0, + "max_expansions" : 50, + "fuzzy_transpositions" : true, + "lenient" : false, + "zero_terms_query" : "NONE", + "auto_generate_synonyms_phrase_query" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -365,16 +376,17 @@ void matchPhraseRelevanceQueryFunction_generatesMatchPhraseQuery() assertInstanceOf(MatchPhraseQueryBuilder.class, result); assertEquals( """ - { - "match_phrase" : { - "b" : { - "query" : "Hi", - "slop" : 2, - "zero_terms_query" : "NONE", - "boost" : 1.0 - } - } - }""", + { + "match_phrase" : { + "b" : { + "query" : "Hi", + "slop" : 2, + "zero_terms_query" : "NONE", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -396,19 +408,20 @@ void matchBoolPrefixRelevanceQueryFunction_generatesMatchBoolPrefixQuery() assertInstanceOf(MatchBoolPrefixQueryBuilder.class, result); assertEquals( """ - { - "match_bool_prefix" : { - "b" : { - "query" : "Hi", - "operator" : "OR", - "minimum_should_match" : "1", - "prefix_length" : 0, - "max_expansions" : 50, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - } - }""", + { + "match_bool_prefix" : { + "b" : { + "query" : "Hi", + "operator" : "OR", + "minimum_should_match" : "1", + "prefix_length" : 0, + "max_expansions" : 50, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -430,18 +443,19 @@ void matchPhrasePrefixRelevanceQueryFunction_generatesMatchPhrasePrefixQuery() assertInstanceOf(MatchPhrasePrefixQueryBuilder.class, result); assertEquals( """ - { - "match_phrase_prefix" : { - "b" : { - "query" : "Hi", - "analyzer" : "standard", - "slop" : 0, - "max_expansions" : 50, - "zero_terms_query" : "NONE", - "boost" : 1.0 - } - } - }""", + { + "match_phrase_prefix" : { + "b" : { + "query" : "Hi", + "analyzer" : "standard", + "slop" : 0, + "max_expansions" : 50, + "zero_terms_query" : "NONE", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -473,27 +487,28 @@ void queryStringRelevanceQueryFunction_generatesQueryStringQuery() assertInstanceOf(QueryStringQueryBuilder.class, result); assertEquals( """ - { - "query_string" : { - "query" : "Hi", - "fields" : [ - "b^1.0", - "c^2.5" - ], - "type" : "best_fields", - "default_operator" : "or", - "max_determinized_states" : 10000, - "enable_position_increments" : true, - "fuzziness" : "1", - "fuzzy_prefix_length" : 0, - "fuzzy_max_expansions" : 50, - "phrase_slop" : 0, - "escape" : false, - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "query_string" : { + "query" : "Hi", + "fields" : [ + "b^1.0", + "c^2.5" + ], + "type" : "best_fields", + "default_operator" : "or", + "max_determinized_states" : 10000, + "enable_position_increments" : true, + "fuzziness" : "1", + "fuzzy_prefix_length" : 0, + "fuzzy_max_expansions" : 50, + "phrase_slop" : 0, + "escape" : false, + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -518,22 +533,23 @@ void simpleQueryStringRelevanceQueryFunction_generatesSimpleQueryStringQuery() assertInstanceOf(SimpleQueryStringBuilder.class, result); assertEquals( """ - { - "simple_query_string" : { - "query" : "Hi", - "fields" : [ - "b*^1.0" - ], - "flags" : -1, - "default_operator" : "or", - "analyze_wildcard" : false, - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_prefix_length" : 0, - "fuzzy_max_expansions" : 50, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "simple_query_string" : { + "query" : "Hi", + "fields" : [ + "b*^1.0" + ], + "flags" : -1, + "default_operator" : "or", + "analyze_wildcard" : false, + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_prefix_length" : 0, + "fuzzy_max_expansions" : 50, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -561,23 +577,24 @@ void multiMatchRelevanceQueryFunction_generatesMultiMatchQuery() assertInstanceOf(MultiMatchQueryBuilder.class, result); assertEquals( """ - { - "multi_match" : { - "query" : "Hi", - "fields" : [ - "b*^1.0" - ], - "type" : "best_fields", - "operator" : "OR", - "slop" : 0, - "prefix_length" : 0, - "max_expansions" : 25, - "zero_terms_query" : "NONE", - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "multi_match" : { + "query" : "Hi", + "fields" : [ + "b*^1.0" + ], + "type" : "best_fields", + "operator" : "OR", + "slop" : 0, + "prefix_length" : 0, + "max_expansions" : 25, + "zero_terms_query" : "NONE", + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -590,15 +607,16 @@ void likeFunction_keywordField_generatesWildcardQuery() throws ExpressionNotAnal assertInstanceOf(WildcardQueryBuilder.class, result); assertEquals( """ - { - "wildcard" : { - "b.keyword" : { - "wildcard" : "*Hi*", - "case_insensitive" : true, - "boost" : 1.0 - } - } - }""", + { + "wildcard" : { + "b.keyword" : { + "wildcard" : "*Hi*", + "case_insensitive" : true, + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -642,54 +660,55 @@ void andOrNot_generatesCompoundQuery() throws ExpressionNotAnalyzableException { assertInstanceOf(BoolQueryBuilder.class, result); assertEquals( """ - { - "bool" : { - "must_not" : [ - { - "bool" : { - "must" : [ - { - "bool" : { - "should" : [ - { - "term" : { - "a" : { - "value" : 12, - "boost" : 1.0 - } - } - }, - { - "term" : { - "a" : { - "value" : 13, - "boost" : 1.0 - } - } + { + "bool" : { + "must_not" : [ + { + "bool" : { + "must" : [ + { + "bool" : { + "should" : [ + { + "term" : { + "a" : { + "value" : 12, + "boost" : 1.0 + } + } + }, + { + "term" : { + "a" : { + "value" : 13, + "boost" : 1.0 } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - { - "term" : { - "b.keyword" : { - "value" : "Hi", - "boost" : 1.0 } } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + { + "term" : { + "b.keyword" : { + "value" : "Hi", + "boost" : 1.0 } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + } } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } } - }""", + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -701,14 +720,15 @@ void equals_generatesTermQuery_TextWithKeyword() throws ExpressionNotAnalyzableE assertInstanceOf(TermQueryBuilder.class, result); assertEquals( """ - { - "term" : { - "b.keyword" : { - "value" : "Hi", - "boost" : 1.0 - } - } - }""", + { + "term" : { + "b.keyword" : { + "value" : "Hi", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -766,14 +786,15 @@ void isTrue_predicate() throws ExpressionNotAnalyzableException { assertInstanceOf(TermQueryBuilder.class, result); assertEquals( """ - { - "term" : { - "b.keyword" : { - "value" : "Hi", - "boost" : 1.0 - } - } - }""", + { + "term" : { + "b.keyword" : { + "value" : "Hi", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -828,22 +849,23 @@ void verify_partial_pushdown() throws ExpressionNotAnalyzableException { assertInstanceOf(BoolQueryBuilder.class, resultBuilder); assertEquals( """ + { + "bool" : { + "must" : [ { - "bool" : { - "must" : [ - { - "term" : { - "a" : { - "value" : 12, - "boost" : 1.0 - } - } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + "term" : { + "a" : { + "value" : 12, + "boost" : 1.0 + } } - }""", + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, resultBuilder.toString()); List unAnalyzableNodes = result.getUnAnalyzableNodes(); @@ -869,22 +891,23 @@ void verify_partial_pushdown() throws ExpressionNotAnalyzableException { assertInstanceOf(BoolQueryBuilder.class, resultBuilder); assertEquals( """ - { - "bool" : { - "must" : [ - { - "term" : { - "a" : { - "value" : 12, - "boost" : 1.0 - } - } + { + "bool" : { + "must" : [ + { + "term" : { + "a" : { + "value" : 12, + "boost" : 1.0 } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + } } - }""", + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, resultBuilder.toString()); } @@ -899,21 +922,22 @@ void multiMatchWithoutFields_generatesMultiMatchQuery() throws ExpressionNotAnal assertInstanceOf(MultiMatchQueryBuilder.class, result); assertEquals( """ - { - "multi_match" : { - "query" : "Hi", - "fields" : [ ], - "type" : "best_fields", - "operator" : "OR", - "slop" : 0, - "prefix_length" : 0, - "max_expansions" : 50, - "zero_terms_query" : "NONE", - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "multi_match" : { + "query" : "Hi", + "fields" : [ ], + "type" : "best_fields", + "operator" : "OR", + "slop" : 0, + "prefix_length" : 0, + "max_expansions" : 50, + "zero_terms_query" : "NONE", + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -930,19 +954,20 @@ void simpleQueryStringWithoutFields_generatesSimpleQueryStringQuery() assertInstanceOf(SimpleQueryStringBuilder.class, result); assertEquals( """ - { - "simple_query_string" : { - "query" : "Hi", - "flags" : -1, - "default_operator" : "or", - "analyze_wildcard" : false, - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_prefix_length" : 0, - "fuzzy_max_expansions" : 50, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "simple_query_string" : { + "query" : "Hi", + "flags" : -1, + "default_operator" : "or", + "analyze_wildcard" : false, + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_prefix_length" : 0, + "fuzzy_max_expansions" : 50, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -959,24 +984,25 @@ void queryStringWithoutFields_generatesQueryStringQuery() assertInstanceOf(QueryStringQueryBuilder.class, result); assertEquals( """ - { - "query_string" : { - "query" : "Hi", - "fields" : [ ], - "type" : "best_fields", - "default_operator" : "or", - "max_determinized_states" : 10000, - "enable_position_increments" : true, - "fuzziness" : "AUTO", - "fuzzy_prefix_length" : 0, - "fuzzy_max_expansions" : 50, - "phrase_slop" : 0, - "escape" : false, - "auto_generate_synonyms_phrase_query" : true, - "fuzzy_transpositions" : true, - "boost" : 1.0 - } - }""", + { + "query_string" : { + "query" : "Hi", + "fields" : [ ], + "type" : "best_fields", + "default_operator" : "or", + "max_determinized_states" : 10000, + "enable_position_increments" : true, + "fuzziness" : "AUTO", + "fuzzy_prefix_length" : 0, + "fuzzy_max_expansions" : 50, + "phrase_slop" : 0, + "escape" : false, + "auto_generate_synonyms_phrase_query" : true, + "fuzzy_transpositions" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -988,18 +1014,19 @@ void equals_generatesRangeQueryForDateTime() throws ExpressionNotAnalyzableExcep assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "d" : { - "from" : "1987-02-03T04:34:56.000Z", - "to" : "1987-02-03T04:34:56.000Z", - "include_lower" : true, - "include_upper" : true, - "format" : "date_time", - "boost" : 1.0 - } - } - }""", + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } @@ -1011,38 +1038,39 @@ void notEquals_generatesBoolQueryForDateTime() throws ExpressionNotAnalyzableExc assertInstanceOf(BoolQueryBuilder.class, result); assertEquals( """ - { - "bool" : { - "should" : [ - { - "range" : { - "d" : { - "from" : "1987-02-03T04:34:56.000Z", - "to" : null, - "include_lower" : false, - "include_upper" : true, - "format" : "date_time", - "boost" : 1.0 - } - } - }, - { - "range" : { - "d" : { - "from" : null, - "to" : "1987-02-03T04:34:56.000Z", - "include_lower" : true, - "include_upper" : false, - "format" : "date_time", - "boost" : 1.0 - } - } + { + "bool" : { + "should" : [ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 + } + }, + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : false, + "format" : "date_time", + "boost" : 1.0 + } + } } - }""", + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, result.toString()); } @@ -1055,18 +1083,19 @@ void gte_generatesRangeQueryWithFormatForDateTime() throws ExpressionNotAnalyzab assertInstanceOf(RangeQueryBuilder.class, result); assertEquals( """ - { - "range" : { - "d" : { - "from" : "1987-02-03T04:34:56.000Z", - "to" : null, - "include_lower" : true, - "include_upper" : true, - "format" : "date_time", - "boost" : 1.0 - } - } - }""", + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, result.toString()); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 2fd12db0296..310bb5e73c5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -1742,7 +1742,8 @@ void cast_to_ip_in_filter(LiteralExpression expr) { "boost" : 1.0 } } - }""", + }\ + """, expr.valueOf().stringValue()); assertJsonEquals(json, buildQuery(DSL.equal(ref("ip_value", IP), DSL.castIp(expr)))); diff --git a/plugin/build.gradle b/plugin/build.gradle index 6d80972edd3..55276b49cc3 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -149,7 +149,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } @@ -284,7 +284,7 @@ afterEvaluate { version = "${project.version}" - "-SNAPSHOT" into '/usr/share/opensearch/plugins' - from(zipTree(bundlePlugin.archivePath)) { + from(zipTree(bundlePlugin.archiveFile)) { into opensearchplugin.name } diff --git a/ppl/build.gradle b/ppl/build.gradle index 3e244c6fa01..ef7973b1e37 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -81,7 +81,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 67403741a82..6678bf85d0e 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -712,11 +712,11 @@ public void testLineCommentShouldPass() { new PPLSyntaxParser() .parse( """ - // test is a new line comment \ - search source=t a=1 b=2 // test is a line comment at the end of ppl command \ - | fields a,b // this is line comment inner ppl command\ - ////this is a new line comment - """)); + // test is a new line comment \ + search source=t a=1 b=2 // test is a line comment at the end of ppl command \ + | fields a,b // this is line comment inner ppl command\ + ////this is a new line comment + """)); } @Test @@ -727,20 +727,20 @@ public void testBlockCommentShouldPass() { new PPLSyntaxParser() .parse( """ - /* + /* + This is a\ + multiple\ + line\ + block\ + comment */\ + search /* block comment */ source=t /* block comment */ a=1 b=2 + |/* This is a\ multiple\ line\ block\ - comment */\ - search /* block comment */ source=t /* block comment */ a=1 b=2 - |/* - This is a\ - multiple\ - line\ - block\ - comment */ fields a,b /* block comment */ \ - """)); + comment */ fields a,b /* block comment */ \ + """)); } @Test diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java index 3b4c2b72a27..5c26d70335b 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLInSubqueryTest.java @@ -245,16 +245,16 @@ public void testInSubqueryAsJoinFilter() { public void failWhenNumOfColumnsNotMatchOutputOfSubquery() { String less = """ - source=EMP | where (DEPTNO) in [ source=DEPT | fields DEPTNO, DNAME ] - | sort - EMPNO | fields EMPNO, ENAME - """; + source=EMP | where (DEPTNO) in [ source=DEPT | fields DEPTNO, DNAME ] + | sort - EMPNO | fields EMPNO, ENAME + """; assertThrows(SemanticCheckException.class, () -> getRelNode(less)); String more = """ - source=EMP | where (DEPTNO, ENAME) in [ source=DEPT | fields DEPTNO, DNAME, LOC ] - | sort - EMPNO | fields EMPNO, ENAME - """; + source=EMP | where (DEPTNO, ENAME) in [ source=DEPT | fields DEPTNO, DNAME, LOC ] + | sort - EMPNO | fields EMPNO, ENAME + """; assertThrows(SemanticCheckException.class, () -> getRelNode(more)); } diff --git a/prometheus/build.gradle b/prometheus/build.gradle index c35269ab452..c3f746e03d9 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -30,6 +30,7 @@ dependencies { testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}" + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } test { diff --git a/protocol/build.gradle b/protocol/build.gradle index fe937648bbe..e6385ab69c2 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -42,6 +42,7 @@ dependencies { testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: "${hamcrest_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}" + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } configurations.all { @@ -86,7 +87,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } } diff --git a/sql/build.gradle b/sql/build.gradle index 2278496a46d..8391d1538d9 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -57,6 +57,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: "${mockito_version}" testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${mockito_version}" testImplementation(testFixtures(project(":core"))) + testRuntimeOnly('org.junit.platform:junit-platform-launcher') } spotless { @@ -73,7 +74,7 @@ spotless { removeUnusedImports() trimTrailingWhitespace() endWithNewline() - googleJavaFormat('1.17.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') + googleJavaFormat('1.32.0').reflowLongStrings().groupArtifact('com.google.googlejavaformat:google-java-format') } }