WIP: merge udf with existing type checking

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

DEVELOPER_GUIDE.rst

@@ -231,6 +231,8 @@ Most of the time you just need to run ./gradlew build which will make sure you p
      - Run all unit tests.
    * - ./gradlew :integ-test:integTest
      - Run all integration test (this takes time).
+   * - ./gradlew :integ-test:yamlRestTest
+     - Run rest integration test.
    * - ./gradlew :doctest:doctest
      - Run doctests
    * - ./gradlew build

build.gradle

@@ -6,9 +6,9 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-beta1-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        buildVersionQualifier = System.getProperty("build.version_qualifier", "beta1")
+        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
         prometheus_binary_version = "2.37.2"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -10,6 +10,7 @@
 import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_LAST;
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC;
 import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC;
+import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
 import static org.opensearch.sql.data.type.ExprCoreType.DATE;
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 import static org.opensearch.sql.data.type.ExprCoreType.TIME;
@@ -53,11 +54,14 @@
 import org.opensearch.sql.ast.tree.FillNull;
 import org.opensearch.sql.ast.tree.Filter;
 import org.opensearch.sql.ast.tree.Head;
+import org.opensearch.sql.ast.tree.Join;
 import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Limit;
+import org.opensearch.sql.ast.tree.Lookup;
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -70,7 +74,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
@@ -162,8 +165,8 @@ public LogicalPlan visitSubqueryAlias(SubqueryAlias node, AnalysisContext contex
           STRUCT);
       return child;
     } else {
-      // TODO
-      throw new UnsupportedOperationException("SubqueryAlias is only supported in table alias");
+      throw new UnsupportedOperationException(
+          "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
     }
   }
 
@@ -490,7 +493,7 @@ public LogicalPlan visitParse(Parse node, AnalysisContext context) {
   }
 
   @Override
-  public LogicalPlan visitWindow(Window node, AnalysisContext context) {
+  public LogicalPlan visitPatterns(Patterns node, AnalysisContext context) {
     LogicalPlan child = node.getChild().get(0).accept(this, context);
     WindowExpressionAnalyzer windowAnalyzer =
         new WindowExpressionAnalyzer(expressionAnalyzer, child);
@@ -685,6 +688,18 @@ public LogicalPlan visitCloseCursor(CloseCursor closeCursor, AnalysisContext con
     return new LogicalCloseCursor(closeCursor.getChild().get(0).accept(this, context));
   }
 
+  @Override
+  public LogicalPlan visitJoin(Join node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Join is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public LogicalPlan visitLookup(Lookup node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Lookup is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
   private LogicalSort buildSort(
       LogicalPlan child, AnalysisContext context, List<Field> sortFields) {
     ExpressionReferenceOptimizer optimizer =

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -7,6 +7,7 @@
 
 import static org.opensearch.sql.ast.dsl.AstDSL.and;
 import static org.opensearch.sql.ast.dsl.AstDSL.compare;
+import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -49,6 +50,9 @@
 import org.opensearch.sql.ast.expression.When;
 import org.opensearch.sql.ast.expression.WindowFunction;
 import org.opensearch.sql.ast.expression.Xor;
+import org.opensearch.sql.ast.expression.subquery.ExistsSubquery;
+import org.opensearch.sql.ast.expression.subquery.InSubquery;
+import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.data.type.ExprType;
@@ -336,7 +340,7 @@ public Expression visitCase(Case node, AnalysisContext context) {
     }
 
     Expression defaultResult =
-        (node.getElseClause() == null) ? null : analyze(node.getElseClause(), context);
+        node.getElseClause().map(elseClause -> analyze(elseClause, context)).orElse(null);
     CaseClause caseClause = new CaseClause(whens, defaultResult);
 
     // To make this simple, require all result type same regardless of implicit convert
@@ -406,6 +410,24 @@ public Expression visitArgument(Argument node, AnalysisContext context) {
     return new NamedArgumentExpression(node.getArgName(), node.getValue().accept(this, context));
   }
 
+  @Override
+  public Expression visitScalarSubquery(ScalarSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public Expression visitExistsSubquery(ExistsSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
+  @Override
+  public Expression visitInSubquery(InSubquery node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Subsearch is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
   /**
    * If QualifiedName is actually a reserved metadata field, return the expr type associated with
    * the metadata field.

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -8,6 +8,7 @@
 import org.opensearch.sql.ast.expression.AggregateFunction;
 import org.opensearch.sql.ast.expression.Alias;
 import org.opensearch.sql.ast.expression.AllFields;
+import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta;
 import org.opensearch.sql.ast.expression.And;
 import org.opensearch.sql.ast.expression.Argument;
 import org.opensearch.sql.ast.expression.AttributeList;
@@ -58,6 +59,7 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
@@ -254,6 +256,10 @@ public T visitAllFields(AllFields node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitAllFieldsExcludeMeta(AllFieldsExcludeMeta node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitNestedAllTupleFields(NestedAllTupleFields node, C context) {
     return visitChildren(node, context);
   }
@@ -338,6 +344,10 @@ public T visitFillNull(FillNull fillNull, C context) {
     return visitChildren(fillNull, context);
   }
 
+  public T visitPatterns(Patterns patterns, C context) {
+    return visitChildren(patterns, context);
+  }
+
   public T visitWindow(Window window, C context) {
     return visitChildren(window, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -58,6 +58,7 @@
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.Parse;
+import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
 import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.RareTopN.CommandType;
@@ -71,7 +72,6 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
-import org.opensearch.sql.ast.tree.Window;
 
 /** Class of static methods to create specific node instances. */
 @UtilityClass
@@ -293,7 +293,7 @@ public UnresolvedExpression caseWhen(UnresolvedExpression elseClause, When... wh
    */
   public UnresolvedExpression caseWhen(
       UnresolvedExpression caseValueExpr, UnresolvedExpression elseClause, When... whenClauses) {
-    return new Case(caseValueExpr, Arrays.asList(whenClauses), elseClause);
+    return new Case(caseValueExpr, Arrays.asList(whenClauses), Optional.ofNullable(elseClause));
   }
 
   public UnresolvedExpression cast(UnresolvedExpression expr, Literal type) {
@@ -504,7 +504,7 @@ public static Parse parse(
     return new Parse(parseMethod, sourceField, pattern, arguments, input);
   }
 
-  public static Window window(
+  public static Patterns patterns(
       UnresolvedPlan input,
       PatternMethod patternMethod,
       UnresolvedExpression sourceField,
@@ -513,7 +513,7 @@ public static Window window(
     List<UnresolvedExpression> funArgs = new ArrayList<>();
     funArgs.add(sourceField);
     funArgs.addAll(arguments);
-    return new Window(
+    return new Patterns(
         new Alias(
             alias,
             new WindowFunction(

core/src/main/java/org/opensearch/sql/ast/expression/Alias.java

@@ -5,23 +5,20 @@
 
 package org.opensearch.sql.ast.expression;
 
-import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /**
  * Alias abstraction that associate an unnamed expression with a name. The name information
  * preserved is useful for semantic analysis and response formatting eventually. This can avoid
  * restoring the info in toString() method which is inaccurate because original info is already
  * lost.
  */
-@AllArgsConstructor
 @EqualsAndHashCode(callSuper = false)
 @Getter
-@RequiredArgsConstructor
 @ToString
 public class Alias extends UnresolvedExpression {
 
@@ -35,7 +32,41 @@ public class Alias extends UnresolvedExpression {
   private final UnresolvedExpression delegated;
 
   /** TODO. Optional field alias. This field is OpenSearch SQL-only */
-  private String alias;
+  private final String alias;
+
+  public Alias(String name, UnresolvedExpression expr) {
+    this(name, expr, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, String alias) {
+    this(name, expr, alias, false);
+  }
+
+  public Alias(String name, UnresolvedExpression expr, boolean metaDataFieldAllowed) {
+    this(name, expr, null, metaDataFieldAllowed);
+  }
+
+  /**
+   * @param metadataFieldAllowed Whether do we allow metadata field as alias name. Should Only be
+   *     true for SQL, see {@link Alias::newAliasAllowMetaMetaField}
+   */
+  private Alias(
+      String name, UnresolvedExpression expr, String alias, boolean metadataFieldAllowed) {
+    if (!metadataFieldAllowed && OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(name)) {
+      throw new IllegalArgumentException(
+          String.format("Cannot use metadata field [%s] as the alias.", name));
+    }
+    this.name = name;
+    this.delegated = expr;
+    this.alias = alias;
+  }
+
+  // TODO: Only for SQL. We never allow metadata field as alias but SQL view all select items as
+  //  alias. Need to remove this tricky logic after SQL fix it.
+  public static Alias newAliasAllowMetaMetaField(
+      String name, UnresolvedExpression expr, String alias) {
+    return new Alias(name, expr, alias, true);
+  }
 
   @Override
   public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {

core/src/main/java/org/opensearch/sql/ast/expression/AllFields.java

@@ -18,7 +18,7 @@
 public class AllFields extends UnresolvedExpression {
   public static final AllFields INSTANCE = new AllFields();
 
-  private AllFields() {}
+  public AllFields() {}
 
   public static AllFields of() {
     return INSTANCE;

core/src/main/java/org/opensearch/sql/ast/expression/AllFieldsExcludeMeta.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.expression;
+
+import java.util.Collections;
+import java.util.List;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.Node;
+
+/**
+ * Represent the All fields but excluding metadata fields if user never uses them in the previous
+ * fields command
+ */
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class AllFieldsExcludeMeta extends AllFields {
+  public static final AllFieldsExcludeMeta INSTANCE = new AllFieldsExcludeMeta();
+
+  private AllFieldsExcludeMeta() {
+    super();
+  }
+
+  public static AllFieldsExcludeMeta of() {
+    return INSTANCE;
+  }
+
+  @Override
+  public List<? extends Node> getChild() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public <R, C> R accept(AbstractNodeVisitor<R, C> nodeVisitor, C context) {
+    return nodeVisitor.visitAllFieldsExcludeMeta(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/expression/Case.java

@@ -7,6 +7,7 @@
 
 import com.google.common.collect.ImmutableList;
 import java.util.List;
+import java.util.Optional;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -31,7 +32,7 @@ public class Case extends UnresolvedExpression {
   private final List<When> whenClauses;
 
   /** Expression that represents ELSE statement result. */
-  private final UnresolvedExpression elseClause;
+  private final Optional<UnresolvedExpression> elseClause;
 
   @Override
   public List<? extends Node> getChild() {
@@ -40,10 +41,7 @@ public List<? extends Node> getChild() {
       children.add(caseValue);
     }
     children.addAll(whenClauses);
-
-    if (elseClause != null) {
-      children.add(elseClause);
-    }
+    elseClause.ifPresent(children::add);
     return children.build();
   }