merge from main

Signed-off-by: xinyual <xinyual@amazon.com>

Author: xinyual

.github/workflows/maven-publish.yml

@@ -5,11 +5,11 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
+  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
 
 jobs:
   build-and-publish-snapshots:
@@ -35,9 +35,13 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
-
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
       - name: publish snapshots to maven
         run: |
-          ./gradlew publishPluginZipPublicationToSnapshotsRepository
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository

.github/workflows/stalled.yml

@@ -21,8 +21,9 @@ jobs:
         with:
           repo-token: ${{ steps.github_app_token.outputs.token }}
           stale-pr-label: 'stalled'
-          stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.'
-          days-before-pr-stale: 30
+          stale-pr-message: 'This PR is stalled because it has been open for 2 weeks with no activity.'
+          days-before-pr-stale: 14
           days-before-issue-stale: -1
           days-before-pr-close: -1
           days-before-issue-close: -1
+          exempt-draft-pr: true

async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java

@@ -72,7 +72,7 @@ public class SparkConstants {
   public static final String PPL_STANDALONE_PACKAGE =
       "org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT";
   public static final String AWS_SNAPSHOT_REPOSITORY =
-      "https://central.sonatype.com/repository/maven-snapshots/";
+      "https://ci.opensearch.org/ci/dbc/snapshots/maven/";
   public static final String GLUE_HIVE_CATALOG_FACTORY_CLASS =
       "com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory";
   public static final String FLINT_DELEGATE_CATALOG =

async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java

@@ -981,7 +981,7 @@ private String constructExpectedSparkSubmitParameterString(
             "spark.hadoop.fs.s3.customAWSCredentialsProvider=com.amazonaws.emr.AssumeRoleAWSCredentialsProvider",
             "spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory",
             "spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT",
-            "spark.jars.repositories=https://central.sonatype.com/repository/maven-snapshots/",
+            "spark.jars.repositories=https://ci.opensearch.org/ci/dbc/snapshots/maven/",
             "spark.emr-serverless.driverEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
             "spark.executorEnv.JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64/",
             "spark.emr-serverless.driverEnv.FLINT_CLUSTER_NAME=TEST_CLUSTER",

build.gradle

@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.3.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.4.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -67,8 +67,7 @@ buildscript {
     repositories {
         mavenLocal()
         mavenCentral()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     }
 
     dependencies {
@@ -93,9 +92,8 @@ apply plugin: 'opensearch.java-agent'
 repositories {
     mavenLocal()
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
-    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
+    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
     maven { url 'https://jitpack.io' }
-    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
 }
 
 spotless {
@@ -158,9 +156,9 @@ subprojects {
     repositories {
         mavenLocal()
         mavenCentral()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
         maven { url 'https://jitpack.io' }
-        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
     }
 }
 

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -33,6 +33,8 @@ public enum Key {
     PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"),
     PPL_VALUES_MAX_LIMIT("plugins.ppl.values.max.limit"),
     PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"),
+    PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"),
+    PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"),
 
     /** Enable Calcite as execution engine */
     CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),
@@ -48,6 +50,8 @@ public enum Key {
     /** Common Settings for SQL and PPL. */
     QUERY_MEMORY_LIMIT("plugins.query.memory_limit"),
     QUERY_SIZE_LIMIT("plugins.query.size_limit"),
+    QUERY_BUCKET_SIZE("plugins.query.buckets"),
+    SEARCH_MAX_BUCKETS("search.max_buckets"),
     ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"),
     DATASOURCES_URI_HOSTS_DENY_LIST("plugins.query.datasources.uri.hosts.denylist"),
     DATASOURCES_LIMIT("plugins.query.datasources.limit"),

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -86,6 +86,7 @@
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
+import org.opensearch.sql.ast.tree.Replace;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
@@ -802,6 +803,11 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con
     return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context));
   }
 
+  @Override
+  public LogicalPlan visitReplace(Replace node, AnalysisContext context) {
+    throw getOnlyForCalciteException("Replace");
+  }
+
   @Override
   public LogicalPlan visitJoin(Join node, AnalysisContext context) {
     throw getOnlyForCalciteException("Join");

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -74,6 +74,7 @@
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
+import org.opensearch.sql.ast.tree.Replace;
 import org.opensearch.sql.ast.tree.Reverse;
 import org.opensearch.sql.ast.tree.Rex;
 import org.opensearch.sql.ast.tree.SPath;
@@ -250,6 +251,10 @@ public T visitRename(Rename node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitReplace(Replace node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitEval(Eval node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/tree/Replace.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import java.util.Set;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import org.jetbrains.annotations.Nullable;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+@Getter
+@Setter
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class Replace extends UnresolvedPlan {
+  private final List<ReplacePair> replacePairs;
+  private final Set<Field> fieldList;
+  @Nullable private UnresolvedPlan child;
+
+  /**
+   * Constructor with multiple pattern/replacement pairs.
+   *
+   * @param replacePairs List of pattern/replacement pairs
+   * @param fieldList Set of fields to apply replacements to
+   */
+  public Replace(List<ReplacePair> replacePairs, Set<Field> fieldList) {
+    this.replacePairs = replacePairs;
+    this.fieldList = fieldList;
+  }
+
+  @Override
+  public Replace attach(UnresolvedPlan child) {
+    if (null == this.child) {
+      this.child = child;
+    } else {
+      this.child.attach(child);
+    }
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitReplace(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/ReplacePair.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.expression.Literal;
+
+/** A pair of pattern and replacement literals for the Replace command. */
+@Getter
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class ReplacePair {
+  private final Literal pattern;
+  private final Literal replacement;
+}