Skip to content

Commit c950e16

Browse files
yuancuyuancu
committed
Merge remote-tracking branch 'origin/main' into issues/4201
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
2 parents b8bb898 + bdd42bc commit c950e16

96 files changed

Lines changed: 1645 additions & 489 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/maven-publish.yml‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,8 @@ on:
55
push:
66
branches:
77
- main
8-
- 1.*
9-
- 2.*
10-
11-
env:
12-
SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
8+
- '[0-9]+.[0-9]+'
9+
- '[0-9]+.x'
1310

1411
jobs:
1512
build-and-publish-snapshots:
@@ -35,9 +32,13 @@ jobs:
3532
export-env: true
3633
env:
3734
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
38-
SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
39-
SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
40-
35+
MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
36+
MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
37+
- name: Configure AWS credentials
38+
uses: aws-actions/configure-aws-credentials@v5
39+
with:
40+
role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
41+
aws-region: us-east-1
4142
- name: publish snapshots to maven
4243
run: |
43-
./gradlew publishPluginZipPublicationToSnapshotsRepository
44+
./gradlew publishPluginZipPublicationToSnapshotsRepository

‎.github/workflows/publish-async-query-core.yml‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ on:
55
push:
66
branches:
77
- main
8-
- 1.*
9-
- 2.*
8+
- '[0-9]+.[0-9]+'
9+
- '[0-9]+.x'
1010
paths:
1111
- 'async-query-core/**'
1212
- '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
1818
cancel-in-progress: false
1919

2020
env:
21-
SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
2221
COMMIT_MAP_FILENAME: commit-history-async-query-core.json
2322

2423
jobs:
@@ -47,8 +46,19 @@ jobs:
4746
export-env: true
4847
env:
4948
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
50-
SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
51-
SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
49+
MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
50+
MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
51+
52+
- name: Export SNAPSHOT_REPO_URL
53+
run: |
54+
snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
55+
echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
56+
57+
- name: Configure AWS credentials
58+
uses: aws-actions/configure-aws-credentials@v5
59+
with:
60+
role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
61+
aws-region: us-east-1
5262

5363
- name: Set commit ID
5464
id: set_commit
@@ -83,4 +93,4 @@ jobs:
8393
source ./.github/maven-publish-utils.sh
8494
8595
# Call the main function for async-query-core
86-
publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
96+
publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

‎.github/workflows/publish-grammar-files.yml‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ on:
55
push:
66
branches:
77
- main
8-
- 1.*
9-
- 2.*
8+
- '[0-9]+.[0-9]+'
9+
- '[0-9]+.x'
1010
paths:
1111
- 'language-grammar/src/main/antlr4/**'
1212
- 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
1919
cancel-in-progress: false
2020

2121
env:
22-
SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
2322
COMMIT_MAP_FILENAME: commit-history-language-grammar.json
2423

2524
jobs:
@@ -51,8 +50,19 @@ jobs:
5150
export-env: true
5251
env:
5352
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
54-
SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
55-
SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
53+
MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
54+
MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
55+
56+
- name: Export SNAPSHOT_REPO_URL
57+
run: |
58+
snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
59+
echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
60+
61+
- name: Configure AWS credentials
62+
uses: aws-actions/configure-aws-credentials@v5
63+
with:
64+
role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
65+
aws-region: us-east-1
5666

5767
- name: Set version
5868
id: set_version

‎async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public class SparkConstants {
7272
public static final String PPL_STANDALONE_PACKAGE =
7373
"org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT";
7474
public static final String AWS_SNAPSHOT_REPOSITORY =
75-
"https://central.sonatype.com/repository/maven-snapshots/";
75+
"https://ci.opensearch.org/ci/dbc/snapshots/maven/";
7676
public static final String GLUE_HIVE_CATALOG_FACTORY_CLASS =
7777
"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory";
7878
public static final String FLINT_DELEGATE_CATALOG =

‎async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -981,7 +981,7 @@ private String constructExpectedSparkSubmitParameterString(
981981
"spark.hadoop.fs.s3.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider",
982982
"spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory",
983983
"spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT",
984-
"spark.jars.repositories=https://central.sonatype.com/repository/maven-snapshots/",
984+
"spark.jars.repositories=https://ci.opensearch.org/ci/dbc/snapshots/maven/",
985985
"spark.emr-serverless.driverEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
986986
"spark.executorEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
987987
"spark.emr-serverless.driverEnv.FLINT_CLUSTER_NAME=TEST_CLUSTER",

‎build.gradle‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
buildscript {
88
ext {
9-
opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT")
9+
opensearch_version = System.getProperty("opensearch.version", "3.4.0-SNAPSHOT")
1010
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
1111
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
1212
version_tokens = opensearch_version.tokenize('-')
@@ -67,8 +67,7 @@ buildscript {
6767
repositories {
6868
mavenLocal()
6969
mavenCentral()
70-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
71-
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
70+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
7271
}
7372

7473
dependencies {
@@ -93,9 +92,8 @@ apply plugin: 'opensearch.java-agent'
9392
repositories {
9493
mavenLocal()
9594
mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
96-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
95+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
9796
maven { url 'https://jitpack.io' }
98-
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
9997
}
10098

10199
spotless {
@@ -158,9 +156,9 @@ subprojects {
158156
repositories {
159157
mavenLocal()
160158
mavenCentral()
161-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
159+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
160+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
162161
maven { url 'https://jitpack.io' }
163-
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
164162
}
165163
}
166164

‎core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
import org.opensearch.sql.ast.tree.Trendline;
8181
import org.opensearch.sql.ast.tree.UnresolvedPlan;
8282
import org.opensearch.sql.ast.tree.Values;
83+
import org.opensearch.sql.calcite.plan.OpenSearchConstants;
8384

8485
/** Class of static methods to create specific node instances. */
8586
@UtilityClass
@@ -491,15 +492,19 @@ public static Span spanFromSpanLengthLiteral(
491492
UnresolvedExpression field, Literal spanLengthLiteral) {
492493
if (spanLengthLiteral.getType() == DataType.STRING) {
493494
String spanText = spanLengthLiteral.getValue().toString();
494-
String valueStr = spanText.replaceAll("[^0-9]", "");
495-
String unitStr = spanText.replaceAll("[0-9]", "");
495+
String valueStr = spanText.replaceAll("[^0-9-]", "");
496+
String unitStr = spanText.replaceAll("[0-9-]", "");
496497

497498
if (valueStr.isEmpty()) {
498499
// No numeric value found, use the literal as-is
499500
return new Span(field, spanLengthLiteral, SpanUnit.NONE);
500501
} else {
501502
// Parse numeric value and unit
502503
Integer value = Integer.parseInt(valueStr);
504+
if (value <= 0) {
505+
throw new IllegalArgumentException(
506+
String.format("Zero or negative time interval not supported: %s", spanText));
507+
}
503508
SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr);
504509
return span(field, intLiteral(value), unit);
505510
}
@@ -713,4 +718,9 @@ public static Bin bin(UnresolvedExpression field, Argument... arguments) {
713718
return DefaultBin.builder().field(field).alias(alias).build();
714719
}
715720
}
721+
722+
/** Get a reference to the implicit timestamp field {@code @timestamp} */
723+
public static Field referImplicitTimestampField() {
724+
return AstDSL.field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP);
725+
}
716726
}

‎core/src/main/java/org/opensearch/sql/ast/statement/Explain.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ public enum ExplainFormat {
3939
SIMPLE,
4040
STANDARD,
4141
EXTENDED,
42-
COST
42+
COST,
43+
/** Formats explain output in yaml format. */
44+
YAML
4345
}
4446

4547
public static ExplainFormat format(String format) {

‎core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,22 @@ public boolean equals(Object o) {
109109
public int hashCode() {
110110
return Objects.hash(root, calcite);
111111
}
112+
113+
public static ExplainResponse normalizeLf(ExplainResponse response) {
114+
ExecutionEngine.ExplainResponseNodeV2 calcite = response.getCalcite();
115+
if (calcite != null) {
116+
return new ExplainResponse(
117+
new ExecutionEngine.ExplainResponseNodeV2(
118+
normalizeLf(calcite.getLogical()),
119+
normalizeLf(calcite.getPhysical()),
120+
normalizeLf(calcite.getExtended())));
121+
}
122+
return response;
123+
}
124+
125+
private static String normalizeLf(String value) {
126+
return value == null ? null : value.replace("\r\n", "\n");
127+
}
112128
}
113129

114130
@AllArgsConstructor

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,8 @@ public void explainWithLegacy(
201201
Explain.ExplainFormat format,
202202
Optional<Throwable> calciteFailure) {
203203
try {
204-
if (format != null && format != Explain.ExplainFormat.STANDARD) {
204+
if (format != null
205+
&& (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) {
205206
throw new UnsupportedOperationException(
206207
"Explain mode " + format.name() + " is not supported in v2 engine");
207208
}

0 commit comments

Comments
 (0)