merge from main

Signed-off-by: xinyual <xinyual@amazon.com>

Author: xinyual

.github/workflows/maven-publish.yml

@@ -29,6 +29,30 @@ jobs:
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
           aws-region: us-east-1
+
+      # Create the initial direct-query directory structure
+      - name: Create direct-query directory structure in repository
+        run: |
+          # Get credentials for publishing
+          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
+          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
+          echo "::add-mask::$SONATYPE_USERNAME"
+          echo "::add-mask::$SONATYPE_PASSWORD"
+          
+          # Create a placeholder file
+          TEMP_DIR=$(mktemp -d)
+          echo "Directory placeholder - $(date)" > "${TEMP_DIR}/.placeholder"
+          
+          # Upload the placeholder file to create the directory structure
+          echo "Creating initial directory structure..."
+          curl -X PUT -u "${SONATYPE_USERNAME}:${SONATYPE_PASSWORD}" \
+            --upload-file "${TEMP_DIR}/.placeholder" \
+            "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/direct-query/.placeholder"
+          
+          # Clean up
+          rm -rf "${TEMP_DIR}"
+          echo "Directory structure created"
+
       - name: publish snapshots to maven
         run: |
           export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)

build.gradle

@@ -6,9 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-beta1-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "beta1")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
         prometheus_binary_version = "2.37.2"

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -31,6 +31,8 @@ public enum Key {
     CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),
     CALCITE_FALLBACK_ALLOWED("plugins.calcite.fallback.allowed"),
     CALCITE_PUSHDOWN_ENABLED("plugins.calcite.pushdown.enabled"),
+    CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR(
+        "plugins.calcite.pushdown.rowcount.estimation.factor"),
 
     /** Query Settings. */
     FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"),

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -61,6 +61,7 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -73,7 +74,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -493,7 +493,7 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) {
   }
 
   @Override
-  public LogicalPlan visitWindow(Window node, AnalysisContext context) {
+  public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
     LogicalPlan child = node.getChild().get(0).accept(this, context);
     WindowExpressionAnalyzer windowAnalyzer =
         new WindowExpressionAnalyzer(expressionAnalyzer, child);
@@ -593,20 +593,22 @@ public LogicalPlan visitFillNull(final FillNull node, final AnalysisContext cont
 
     ImmutableList.Builder<Pair<ReferenceExpression, Expression>> expressionsBuilder =
         new Builder<>();
-    for (FillNull.NullableFieldFill fieldFill : node.getNullableFieldFills()) {
-      Expression fieldExpr =
-          expressionAnalyzer.analyze(fieldFill.getNullableFieldReference(), context);
+    for (Pair<Field, UnresolvedExpression> fieldFill : node.getReplacementPairs()) {
+      Expression fieldExpr = expressionAnalyzer.analyze(fieldFill.getLeft(), context);
       ReferenceExpression ref =
-          DSL.ref(fieldFill.getNullableFieldReference().getField().toString(), fieldExpr.type());
+          DSL.ref(fieldFill.getLeft().getField().toString(), fieldExpr.type());
       FunctionExpression ifNullFunction =
-          DSL.ifnull(ref, expressionAnalyzer.analyze(fieldFill.getReplaceNullWithMe(), context));
+          DSL.ifnull(ref, expressionAnalyzer.analyze(fieldFill.getRight(), context));
       expressionsBuilder.add(new ImmutablePair<>(ref, ifNullFunction));
       TypeEnvironment typeEnvironment = context.peek();
       // define the new reference in type env.
       typeEnvironment.define(ref);
     }
-
-    return new LogicalEval(child, expressionsBuilder.build());
+    List<Pair<ReferenceExpression, Expression>> expressions = expressionsBuilder.build();
+    if (expressions.isEmpty()) {
+      throw new SemanticCheckException("At least one field is required for fillnull in V2.");
+    }
+    return new LogicalEval(child, expressions);
   }
 
   /** Build {@link LogicalML} for ml command. */

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -60,6 +60,7 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -348,6 +349,10 @@ public T visitFillNull(FillNull fillNull, C context) {
     return visitChildren(fillNull, context);
   }
 
+  public T visitPatterns(Patterns patterns, C context) {
+    return visitChildren(patterns, context);
+  }
+
   public T visitWindow(Window window, C context) {
     return visitChildren(window, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -13,7 +13,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import lombok.experimental.UtilityClass;
-import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.opensearch.sql.ast.expression.AggregateFunction;
 import org.opensearch.sql.ast.expression.Alias;
@@ -58,6 +57,7 @@
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.RareTopN.CommandType;
@@ -71,7 +71,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 
 /** Class of static methods to create specific node instances. */
 @UtilityClass
@@ -504,7 +503,7 @@ public static Parse parse(
     return new Parse(parseMethod, sourceField, pattern, arguments, input);
   }
 
-  public static Window window(
+  public static Patterns patterns(
       UnresolvedPlan input,
       PatternMethod patternMethod,
       UnresolvedExpression sourceField,
@@ -513,7 +512,7 @@ public static Window window(
     List<UnresolvedExpression> funArgs = new ArrayList<>();
     funArgs.add(sourceField);
     funArgs.addAll(arguments);
-    return new Window(
+    return new Patterns(
         new Alias(
             alias,
             new WindowFunction(
@@ -524,21 +523,23 @@ public static Window window(
         input);
   }
 
-  public static FillNull fillNull(UnresolvedExpression replaceNullWithMe, Field... fields) {
-    return new FillNull(
-        FillNull.ContainNullableFieldFill.ofSameValue(
-            replaceNullWithMe, ImmutableList.copyOf(fields)));
+  public static FillNull fillNull(UnresolvedPlan input, UnresolvedExpression replacement) {
+    return FillNull.ofSameValue(replacement, ImmutableList.of()).attach(input);
   }
 
   public static FillNull fillNull(
-      List<ImmutablePair<Field, UnresolvedExpression>> fieldAndReplacements) {
-    ImmutableList.Builder<FillNull.NullableFieldFill> replacementsBuilder = ImmutableList.builder();
-    for (ImmutablePair<Field, UnresolvedExpression> fieldAndReplacement : fieldAndReplacements) {
+      UnresolvedPlan input, UnresolvedExpression replacement, Field... fields) {
+    return FillNull.ofSameValue(replacement, ImmutableList.copyOf(fields)).attach(input);
+  }
+
+  public static FillNull fillNull(
+      UnresolvedPlan input, List<Pair<Field, UnresolvedExpression>> fieldAndReplacements) {
+    ImmutableList.Builder<Pair<Field, UnresolvedExpression>> replacementsBuilder =
+        ImmutableList.builder();
+    for (Pair<Field, UnresolvedExpression> fieldAndReplacement : fieldAndReplacements) {
       replacementsBuilder.add(
-          new FillNull.NullableFieldFill(
-              fieldAndReplacement.getLeft(), fieldAndReplacement.getRight()));
+          Pair.of(fieldAndReplacement.getLeft(), fieldAndReplacement.getRight()));
     }
-    return new FillNull(
-        FillNull.ContainNullableFieldFill.ofVariousValue(replacementsBuilder.build()));
+    return FillNull.ofVariousValue(replacementsBuilder.build()).attach(input);
   }
 }

core/src/main/java/org/opensearch/sql/ast/expression/ParseMethod.java

@@ -11,7 +11,8 @@
 @RequiredArgsConstructor
 public enum ParseMethod {
   REGEX("regex"),
-  GROK("grok");
+  GROK("grok"),
+  PATTERNS("patterns");
 
   @Getter private final String name;
 }

core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import lombok.Getter;
+
+public abstract class WindowBound {
+  private WindowBound() {}
+
+  @Getter
+  public static class OffSetWindowBound extends WindowBound {
+    private final long offset;
+    private final boolean isPreceding;
+
+    OffSetWindowBound(long offset, boolean isPreceding) {
+      this.offset = offset;
+      this.isPreceding = isPreceding;
+    }
+
+    public boolean isPreceding() {
+      return isPreceding;
+    }
+  }
+
+  public static class CurrentRowWindowBound extends WindowBound {
+    CurrentRowWindowBound() {}
+
+    @Override
+    public String toString() {
+      return "CURRENT ROW";
+    }
+  }
+
+  public static class UnboundedWindowBound extends WindowBound {
+    private final boolean isPreceding;
+
+    UnboundedWindowBound(boolean isPreceding) {
+      this.isPreceding = isPreceding;
+    }
+
+    public boolean isPreceding() {
+      return isPreceding;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      return this == o
+          || o instanceof UnboundedWindowBound
+              && isPreceding == ((UnboundedWindowBound) o).isPreceding;
+    }
+
+    @Override
+    public String toString() {
+      return isPreceding ? "UNBOUNDED PRECEDING" : "UNBOUNDED FOLLOWING";
+    }
+  }
+}

core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.Locale;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.ToString;
+
+@EqualsAndHashCode(callSuper = false)
+@Getter
+@RequiredArgsConstructor
+@ToString
+public class WindowFrame extends UnresolvedExpression {
+  private final FrameType type;
+  private final WindowBound lower;
+  private final WindowBound upper;
+
+  public enum FrameType {
+    RANGE,
+    ROWS
+  }
+
+  public static WindowFrame defaultFrame() {
+    return new WindowFrame(
+        FrameType.ROWS, createBound("UNBOUNDED PRECEDING"), createBound("UNBOUNDED FOLLOWING"));
+  }
+
+  public static WindowFrame create(FrameType type, Literal lower, Literal upper) {
+    WindowBound lowerBound = null;
+    WindowBound upperBound = null;
+    if (lower != null) {
+      if (lower.getType() == DataType.STRING) {
+        lowerBound = createBound(lower.getValue().toString());
+      } else {
+        throw new IllegalArgumentException(
+            String.format("Unsupported bound type: %s", lower.getType()));
+      }
+    }
+    if (upper != null) {
+      if (upper.getType() == DataType.STRING) {
+        upperBound = createBound(upper.getValue().toString());
+      } else {
+        throw new IllegalArgumentException(
+            String.format("Unsupported bound type: %s", upper.getType()));
+      }
+    }
+    return new WindowFrame(type, lowerBound, upperBound);
+  }
+
+  private static WindowBound createBound(String boundType) {
+    boundType = boundType.trim().toUpperCase(Locale.ROOT);
+    if ("CURRENT ROW".equals(boundType)) {
+      return new WindowBound.CurrentRowWindowBound();
+    } else if ("UNBOUNDED PRECEDING".equals(boundType)) {
+      return new WindowBound.UnboundedWindowBound(true);
+    } else if ("UNBOUNDED FOLLOWING".equals(boundType)) {
+      return new WindowBound.UnboundedWindowBound(false);
+    } else if (boundType.endsWith(" PRECEDING")) {
+      long number = Long.parseLong(boundType.split(" PRECEDING")[0]);
+      return new WindowBound.OffSetWindowBound(number, true);
+    } else if (boundType.endsWith(" FOLLOWING")) {
+      long number = Long.parseLong(boundType.split(" FOLLOWING")[0]);
+      return new WindowBound.OffSetWindowBound(number, false);
+    } else {
+      throw new IllegalArgumentException(String.format("Unsupported bound type: %s", boundType));
+    }
+  }
+}