Merge remote-tracking branch 'origin/main' into issues/4063

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

.gitignore

@@ -52,5 +52,7 @@ http-client.env.json
 /doctest/opensearch-job-scheduler/
 .factorypath
 
-# Claude Code files
-.claude/
+# Coding agent files (could be symlinks)
+.claude
+.clinerules
+memory-bank

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -30,13 +30,15 @@ public enum Key {
     PATTERN_MAX_SAMPLE_COUNT("plugins.ppl.pattern.max.sample.count"),
     PATTERN_BUFFER_LIMIT("plugins.ppl.pattern.buffer.limit"),
     PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"),
+    PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"),
 
     /** Enable Calcite as execution engine */
     CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),
     CALCITE_FALLBACK_ALLOWED("plugins.calcite.fallback.allowed"),
     CALCITE_PUSHDOWN_ENABLED("plugins.calcite.pushdown.enabled"),
     CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR(
         "plugins.calcite.pushdown.rowcount.estimation.factor"),
+    CALCITE_SUPPORT_ALL_JOIN_TYPES("plugins.calcite.all_join_types.allowed"),
 
     /** Query Settings. */
     FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"),

core/build.gradle

@@ -54,6 +54,7 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
     api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}"
     api group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     api group: 'com.tdunning', name: 't-digest', version: '3.3'
     api "net.minidev:json-smart:${versions.json_smart}"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -880,7 +880,11 @@ private LogicalAggregation analyzeAggregation(
     groupBys.forEach(
         group ->
             newEnv.define(new Symbol(Namespace.FIELD_NAME, group.getNameOrAlias()), group.type()));
-    return new LogicalAggregation(child, aggregators, groupBys);
+
+    Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList());
+    boolean bucketNullable =
+        (Boolean) statsArgs.getOrDefault(Argument.BUCKET_NULLABLE, Literal.TRUE).getValue();
+    return new LogicalAggregation(child, aggregators, groupBys, bucketNullable);
   }
 
   private Aggregation analyzePatternsAgg(Patterns node) {

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -449,6 +449,7 @@ public static List<Argument> defaultStatsArgs() {
         argument("partitions", intLiteral(1)),
         argument("allnum", booleanLiteral(false)),
         argument("delim", stringLiteral(" ")),
+        argument(Argument.BUCKET_NULLABLE, booleanLiteral(true)),
         argument("dedupsplit", booleanLiteral(false)));
   }
 

core/src/main/java/org/opensearch/sql/ast/expression/Argument.java

@@ -20,6 +20,8 @@
 @RequiredArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 public class Argument extends UnresolvedExpression {
+  public static final String BUCKET_NULLABLE = "bucket_nullable";
+
   private final String argName;
   private final Literal value;
 
@@ -39,15 +41,24 @@ public static class ArgumentMap {
     private final Map<String, Literal> map;
 
     public ArgumentMap(List<Argument> arguments) {
-      this.map =
-          arguments.stream()
-              .collect(java.util.stream.Collectors.toMap(Argument::getArgName, Argument::getValue));
+      if (arguments == null || arguments.isEmpty()) {
+        this.map = Map.of();
+      } else {
+        this.map =
+            arguments.stream()
+                .collect(
+                    java.util.stream.Collectors.toMap(Argument::getArgName, Argument::getValue));
+      }
     }
 
     public static ArgumentMap of(List<Argument> arguments) {
       return new ArgumentMap(arguments);
     }
 
+    public static ArgumentMap empty() {
+      return new ArgumentMap(null);
+    }
+
     /**
      * Get argument value by name.
      *
@@ -57,5 +68,9 @@ public static ArgumentMap of(List<Argument> arguments) {
     public Literal get(String name) {
       return map.get(name);
     }
+
+    public Literal getOrDefault(String name, Literal literal) {
+      return map.getOrDefault(name, literal);
+    }
   }
 }

core/src/main/java/org/opensearch/sql/ast/expression/Literal.java

@@ -46,4 +46,9 @@ public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) {
   public String toString() {
     return String.valueOf(value);
   }
+
+  public static Literal TRUE = new Literal(true, DataType.BOOLEAN);
+  public static Literal FALSE = new Literal(false, DataType.BOOLEAN);
+  public static Literal ZERO = new Literal(0, DataType.INTEGER);
+  public static Literal ONE = new Literal(1, DataType.INTEGER);
 }

core/src/main/java/org/opensearch/sql/ast/tree/Join.java

@@ -15,6 +15,8 @@
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Argument;
+import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
 @ToString
@@ -28,20 +30,26 @@ public class Join extends UnresolvedPlan {
   private final JoinType joinType;
   private final Optional<UnresolvedExpression> joinCondition;
   private final JoinHint joinHint;
+  private final Optional<List<Field>> joinFields;
+  private final Argument.ArgumentMap argumentMap;
 
   public Join(
       UnresolvedPlan right,
       Optional<String> leftAlias,
       Optional<String> rightAlias,
       JoinType joinType,
       Optional<UnresolvedExpression> joinCondition,
-      JoinHint joinHint) {
+      JoinHint joinHint,
+      Optional<List<Field>> joinFields,
+      Argument.ArgumentMap argumentMap) {
     this.right = right;
     this.leftAlias = leftAlias;
     this.rightAlias = rightAlias;
     this.joinType = joinType;
     this.joinCondition = joinCondition;
     this.joinHint = joinHint;
+    this.joinFields = joinFields;
+    this.argumentMap = argumentMap;
   }
 
   @Override
@@ -89,6 +97,11 @@ public enum JoinType {
     FULL
   }
 
+  /** RIGHT, CROSS, FULL are performance sensitive join types */
+  public static List<JoinType> highCostJoinTypes() {
+    return List.of(JoinType.RIGHT, JoinType.CROSS, JoinType.FULL);
+  }
+
   @Getter
   @RequiredArgsConstructor
   public static class JoinHint {

core/src/main/java/org/opensearch/sql/ast/tree/Rex.java

@@ -23,38 +23,52 @@
 public class Rex extends UnresolvedPlan {
 
   public enum RexMode {
-    EXTRACT
+    EXTRACT,
+    SED
   }
 
   /** Field to extract from. */
   private final UnresolvedExpression field;
 
-  /** Pattern with named capture groups. */
+  /** Pattern with named capture groups or sed expression. */
   private final Literal pattern;
 
-  /** Rex mode (only EXTRACT supported). */
+  /** Rex mode (extract or sed). */
   private final RexMode mode;
 
   /** Maximum number of matches (optional). */
   private final Optional<Integer> maxMatch;
 
+  /** Offset field name for position tracking (optional). */
+  private final Optional<String> offsetField;
+
   /** Child Plan. */
   @Setter private UnresolvedPlan child;
 
   public Rex(UnresolvedExpression field, Literal pattern) {
-    this(field, pattern, RexMode.EXTRACT, Optional.empty());
+    this(field, pattern, RexMode.EXTRACT, Optional.empty(), Optional.empty());
   }
 
   public Rex(UnresolvedExpression field, Literal pattern, Optional<Integer> maxMatch) {
-    this(field, pattern, RexMode.EXTRACT, maxMatch);
+    this(field, pattern, RexMode.EXTRACT, maxMatch, Optional.empty());
   }
 
   public Rex(
       UnresolvedExpression field, Literal pattern, RexMode mode, Optional<Integer> maxMatch) {
+    this(field, pattern, mode, maxMatch, Optional.empty());
+  }
+
+  public Rex(
+      UnresolvedExpression field,
+      Literal pattern,
+      RexMode mode,
+      Optional<Integer> maxMatch,
+      Optional<String> offsetField) {
     this.field = field;
     this.pattern = pattern;
     this.mode = mode;
     this.maxMatch = maxMatch;
+    this.offsetField = offsetField;
   }
 
   @Override