Merge origin/main into crypto-udf

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

build.gradle

@@ -6,9 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-beta1-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "beta1")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
         prometheus_binary_version = "2.37.2"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -10,6 +10,7 @@
 import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST;
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC;
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC;
+import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
 import static org.opensearch.sql.data.type.ExprCoreType.DATE;
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 import static org.opensearch.sql.data.type.ExprCoreType.TIME;
@@ -53,11 +54,14 @@
 import org.opensearch.sql.ast.tree.FillNull;
 import org.opensearch.sql.ast.tree.Filter;
 import org.opensearch.sql.ast.tree.Head;
+import org.opensearch.sql.ast.tree.Join;
 import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Limit;
+import org.opensearch.sql.ast.tree.Lookup;
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -70,7 +74,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -162,8 +165,8 @@ public LogicalPlan visitSubqueryAlias(SubqueryAlias node, AnalysisContext contex
           STRUCT);
       return child;
     } else {
-      // TODO
-      throw new UnsupportedOperationException("SubqueryAlias is only supported in table alias");
+      throw new UnsupportedOperationException(
+          "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
     }
   }
 
@@ -490,7 +493,7 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) {
   }
 
   @Override
-  public LogicalPlan visitWindow(Window node, AnalysisContext context) {
+  public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
     LogicalPlan child = node.getChild().get(0).accept(this, context);
     WindowExpressionAnalyzer windowAnalyzer =
         new WindowExpressionAnalyzer(expressionAnalyzer, child);
@@ -685,6 +688,18 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con
     return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context));
   }
 
+  @Override
+  public LogicalPlan visitJoin(Join node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Join is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public LogicalPlan visitLookup(Lookup node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Lookup is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
   private LogicalSort buildSort(
       LogicalPlan child, AnalysisContext context, List<Field> sortFields) {
     ExpressionReferenceOptimizer optimizer =

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -7,6 +7,7 @@
 
 import static org.opensearch.sql.ast.dsl.AstDSL.and;
 import static org.opensearch.sql.ast.dsl.AstDSL.compare;
+import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -49,6 +50,9 @@
 import org.opensearch.sql.ast.expression.When;
 import org.opensearch.sql.ast.expression.WindowFunction;
 import org.opensearch.sql.ast.expression.Xor;
+import org.opensearch.sql.ast.expression.subquery.ExistsSubquery;
+import org.opensearch.sql.ast.expression.subquery.InSubquery;
+import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
@@ -336,7 +340,7 @@ public Expression visitCase(Case node, AnalysisContext context) {
     }
 
     Expression defaultResult =
-        (node.getElseClause() == null) ? null : analyze(node.getElseClause(), context);
+        node.getElseClause().map(elseClause -> analyze(elseClause, context)).orElse(null);
     CaseClause caseClause = new CaseClause(whens, defaultResult);
 
     // To make this simple, require all result type same regardless of implicit convert
@@ -406,6 +410,24 @@ public Expression visitArgument(Argument node, AnalysisContext context) {
     return new NamedArgumentExpression(node.getArgName(), node.getValue().accept(this, context));
   }
 
+  @Override
+  public Expression visitScalarSubquery(ScalarSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public Expression visitExistsSubquery(ExistsSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public Expression visitInSubquery(InSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
   /**
    * If QualifiedName is actually a reserved metadata field, return the expr type associated with
    * the metadata field.

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -59,6 +59,7 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -69,7 +70,6 @@
 import org.opensearch.sql.ast.tree.TableFunction;
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 
 /** AST nodes visitor Defines the traverse path. */
 public abstract class AbstractNodeVisitor<T, C> {
@@ -343,8 +343,8 @@ public T visitFillNull(FillNull fillNull, C context) {
     return visitChildren(fillNull, context);
   }
 
-  public T visitWindow(Window window, C context) {
-    return visitChildren(window, context);
+  public T visitPatterns(Patterns patterns, C context) {
+    return visitChildren(patterns, context);
   }
 
   public T visitJoin(Join node, C context) {

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -58,6 +58,7 @@
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.RareTopN.CommandType;
@@ -71,7 +72,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 
 /** Class of static methods to create specific node instances. */
 @UtilityClass
@@ -293,7 +293,7 @@ public UnresolvedExpression caseWhen(UnresolvedExpression elseClause, When... wh
    */
   public UnresolvedExpression caseWhen(
       UnresolvedExpression caseValueExpr, UnresolvedExpression elseClause, When... whenClauses) {
-    return new Case(caseValueExpr, Arrays.asList(whenClauses), elseClause);
+    return new Case(caseValueExpr, Arrays.asList(whenClauses), Optional.ofNullable(elseClause));
   }
 
   public UnresolvedExpression cast(UnresolvedExpression expr, Literal type) {
@@ -504,7 +504,7 @@ public static Parse parse(
     return new Parse(parseMethod, sourceField, pattern, arguments, input);
   }
 
-  public static Window window(
+  public static Patterns patterns(
       UnresolvedPlan input,
       PatternMethod patternMethod,
       UnresolvedExpression sourceField,
@@ -513,7 +513,7 @@ public static Window window(
     List<UnresolvedExpression> funArgs = new ArrayList<>();
     funArgs.add(sourceField);
     funArgs.addAll(arguments);
-    return new Window(
+    return new Patterns(
         new Alias(
             alias,
             new WindowFunction(

core/src/main/java/org/opensearch/sql/ast/expression/Case.java

@@ -7,6 +7,7 @@
 
 import com.google.common.collect.ImmutableList;
 import java.util.List;
+import java.util.Optional;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -31,7 +32,7 @@ public class Case extends UnresolvedExpression {
   private final List<When> whenClauses;
 
   /** Expression that represents ELSE statement result. */
-  private final UnresolvedExpression elseClause;
+  private final Optional<UnresolvedExpression> elseClause;
 
   @Override
   public List<? extends Node> getChild() {
@@ -40,10 +41,7 @@ public List<? extends Node> getChild() {
       children.add(caseValue);
     }
     children.addAll(whenClauses);
-
-    if (elseClause != null) {
-      children.add(elseClause);
-    }
+    elseClause.ifPresent(children::add);
     return children.build();
   }
 

core/src/main/java/org/opensearch/sql/ast/expression/ParseMethod.java

@@ -11,7 +11,8 @@
 @RequiredArgsConstructor
 public enum ParseMethod {
   REGEX("regex"),
-  GROK("grok");
+  GROK("grok"),
+  PATTERNS("patterns");
 
   @Getter private final String name;
 }

core/src/main/java/org/opensearch/sql/ast/statement/Explain.java

@@ -40,7 +40,8 @@ public <R, C> R accept(AbstractNodeVisitor<R, C> visitor, C context) {
   public enum ExplainFormat {
     SIMPLE,
     STANDARD,
-    EXTENDED
+    EXTENDED,
+    COST
   }
 
   public static ExplainFormat format(String format) {

…/org/opensearch/sql/ast/tree/Window.java …rg/opensearch/sql/ast/tree/Patterns.javacore/src/main/java/org/opensearch/sql/ast/tree/Window.java renamed to core/src/main/java/org/opensearch/sql/ast/tree/Patterns.java core/src/main/java/org/opensearch/sql/ast/tree/Window.java renamed to core/src/main/java/org/opensearch/sql/ast/tree/Patterns.java

@@ -16,20 +16,24 @@
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 
+/**
+ * Logical plan node of Patterns command to represent complex nested function calling than single
+ * window function.
+ */
 @Getter
 @Setter
 @ToString
 @EqualsAndHashCode(callSuper = false)
 @RequiredArgsConstructor
 @AllArgsConstructor
-public class Window extends UnresolvedPlan {
+public class Patterns extends UnresolvedPlan {
 
   private final UnresolvedExpression windowFunction;
 
   private UnresolvedPlan child;
 
   @Override
-  public Window attach(UnresolvedPlan child) {
+  public Patterns attach(UnresolvedPlan child) {
     this.child = child;
     return this;
   }
@@ -41,6 +45,6 @@ public List<UnresolvedPlan> getChild() {
 
   @Override
   public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
-    return nodeVisitor.visitWindow(this, context);
+    return nodeVisitor.visitPatterns(this, context);
   }
 }

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -12,6 +12,7 @@
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC;
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import java.util.ArrayList;
@@ -31,8 +32,8 @@
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexWindowBounds;
-import org.apache.calcite.sql.fun.SqlLibraryOperators;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.tools.RelBuilder.AggCall;
 import org.apache.calcite.util.Holder;
@@ -78,6 +79,7 @@
 import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
 import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
+import org.opensearch.sql.expression.function.PPLFuncImpTable;
 import org.opensearch.sql.utils.ParseUtils;
 
 public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalcitePlanContext> {
@@ -283,21 +285,28 @@ public RelNode visitHead(Head node, CalcitePlanContext context) {
   @Override
   public RelNode visitParse(Parse node, CalcitePlanContext context) {
     visitChildren(node, context);
-    List<String> originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames();
     RexNode sourceField = rexVisitor.analyze(node.getSourceField(), context);
     ParseMethod parseMethod = node.getParseMethod();
     java.util.Map<String, Literal> arguments = node.getArguments();
-    String pattern = (String) node.getPattern().getValue();
+    String patternValue = (String) node.getPattern().getValue();
+    String pattern =
+        ParseMethod.PATTERNS.equals(parseMethod) && Strings.isNullOrEmpty(patternValue)
+            ? "[a-zA-Z0-9]"
+            : patternValue;
     List<String> groupCandidates =
         ParseUtils.getNamedGroupCandidates(parseMethod, pattern, arguments);
     List<RexNode> newFields =
         groupCandidates.stream()
             .map(
                 group ->
-                    context.rexBuilder.makeCall(
-                        SqlLibraryOperators.REGEXP_EXTRACT,
+                    PPLFuncImpTable.INSTANCE.resolve(
+                        context.rexBuilder,
+                        ParseUtils.BUILTIN_FUNCTION_MAP.get(parseMethod),
                         sourceField,
-                        context.rexBuilder.makeLiteral(pattern)))
+                        context.rexBuilder.makeLiteral(
+                            pattern,
+                            context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR),
+                            true)))
             .toList();
     projectPlusOverriding(newFields, groupCandidates, context);
     return context.relBuilder.peek();