Merge remote-tracking branch 'origin/main' into pushdown-limit

Author: yuancu

core/src/main/java/org/opensearch/sql/ast/statement/Explain.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.sql.ast.statement;
 
+import java.util.Locale;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
@@ -46,7 +47,7 @@ public enum ExplainFormat {
 
   public static ExplainFormat format(String format) {
     try {
-      return ExplainFormat.valueOf(format.toUpperCase());
+      return ExplainFormat.valueOf(format.toUpperCase(Locale.ROOT));
     } catch (Exception e) {
       return ExplainFormat.STANDARD;
     }

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -37,6 +37,7 @@
 import org.apache.calcite.tools.RelBuilder;
 import org.apache.calcite.tools.RelBuilder.AggCall;
 import org.apache.calcite.util.Holder;
+import org.apache.calcite.util.Pair;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.Node;
@@ -78,6 +79,7 @@
 import org.opensearch.sql.ast.tree.Window;
 import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
+import org.opensearch.sql.calcite.utils.PlanUtils;
 import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.function.PPLFuncImpTable;
@@ -371,10 +373,9 @@ private void projectPlusOverriding(
     context.relBuilder.rename(expectedRenameFields);
   }
 
-  @Override
-  public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) {
-    visitChildren(node, context);
-    List<AggCall> aggList =
+  private Pair<List<AggCall>, List<RexNode>> resolveAggCallAndGroupBy(
+      Aggregation node, CalcitePlanContext context) {
+    List<AggCall> aggCallList =
         node.getAggExprList().stream()
             .map(expr -> aggVisitor.analyze(expr, context))
             .collect(Collectors.toList());
@@ -389,7 +390,37 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) {
     }
     groupByList.addAll(
         node.getGroupExprList().stream().map(expr -> rexVisitor.analyze(expr, context)).toList());
+    return Pair.of(aggCallList, groupByList);
+  }
 
+  @Override
+  public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) {
+    visitChildren(node, context);
+    // Add a trimmed Project before Aggregate.
+    // to avoid bugs in RelDecorrelator.decorrelateRel(Aggregate rel)
+    // For example:
+    // source=t | where a > 1 | stats avg(b+1) by c
+    // Before:
+    // Aggregate
+    //     \- Filter(a>1)
+    //            \- Scan t
+    // After:
+    // Aggregate
+    //     \- Project([c,b])
+    //            \- Filter(a>1)
+    //                   \- Scan t
+    Pair<List<AggCall>, List<RexNode>> resolved = resolveAggCallAndGroupBy(node, context);
+    List<RexInputRef> trimmedRefs = new ArrayList<>();
+    trimmedRefs.addAll(PlanUtils.getInputRefs(resolved.right)); // group-by keys first
+    trimmedRefs.addAll(PlanUtils.getInputRefsFromAggCall(resolved.left));
+    context.relBuilder.project(trimmedRefs);
+
+    // Re-resolve aggCalls and group-by list based on adding trimmed Project.
+    // Using re-resolving rather than Calcite Mapping (ref Calcite ProjectTableScanRule)
+    // because that Mapping only works for RexNode, but we need both AggCall and RexNode list.
+    Pair<List<AggCall>, List<RexNode>> reResolved = resolveAggCallAndGroupBy(node, context);
+    List<AggCall> aggList = reResolved.left;
+    List<RexNode> groupByList = reResolved.right;
     context.relBuilder.aggregate(context.relBuilder.groupKey(groupByList), aggList);
 
     // schema reordering

core/src/main/java/org/opensearch/sql/calcite/plan/AbstractOpenSearchTable.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.calcite.plan;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.schema.TranslatableTable;
+import org.apache.calcite.schema.impl.AbstractTable;
+import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
+
+/**
+ * Abstract class to map the {@link org.opensearch.sql.storage.Table} and {@link
+ * org.apache.calcite.schema.Table}
+ */
+public abstract class AbstractOpenSearchTable extends AbstractTable
+    implements TranslatableTable, org.opensearch.sql.storage.Table {
+
+  @Override
+  public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) {
+    return OpenSearchTypeFactory.convertSchema(this);
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchQueryable.java

@@ -33,6 +33,7 @@
 import java.lang.reflect.Type;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -308,7 +309,7 @@ public static ExprValue getExprValueByExprType(ExprType type, Object value) {
   public static RelDataType convertSchema(Table table) {
     List<String> fieldNameList = new ArrayList<>();
     List<RelDataType> typeList = new ArrayList<>();
-    Map<String, ExprType> fieldTypes = table.getFieldTypes();
+    Map<String, ExprType> fieldTypes = new LinkedHashMap<>(table.getFieldTypes());
     fieldTypes.putAll(table.getReservedFieldTypes());
     for (Entry<String, ExprType> entry : fieldTypes.entrySet()) {
       fieldNameList.add(entry.getKey());

core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchTable.java

@@ -20,7 +20,9 @@
 import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Nullable;
+import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.rex.RexWindowBound;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.type.ReturnTypes;
@@ -259,4 +261,34 @@ static RelBuilder.AggCall makeAggCall(
             "Unexpected aggregation: " + functionName.getName().getFunctionName());
     }
   }
+
+  /** Get all uniq input references from a RexNode. */
+  static List<RexInputRef> getInputRefs(RexNode node) {
+    List<RexInputRef> inputRefs = new ArrayList<>();
+    node.accept(
+        new RexVisitorImpl<Void>(true) {
+          @Override
+          public Void visitInputRef(RexInputRef inputRef) {
+            if (!inputRefs.contains(inputRef)) {
+              inputRefs.add(inputRef);
+            }
+            return null;
+          }
+        });
+    return inputRefs;
+  }
+
+  /** Get all uniq input references from a list of RexNodes. */
+  static List<RexInputRef> getInputRefs(List<RexNode> nodes) {
+    return nodes.stream().flatMap(node -> getInputRefs(node).stream()).toList();
+  }
+
+  /** Get all uniq input references from a list of agg calls. */
+  static List<RexInputRef> getInputRefsFromAggCall(List<RelBuilder.AggCall> aggCalls) {
+    return aggCalls.stream()
+        .map(RelBuilder.AggCall::over)
+        .map(RelBuilder.OverCall::toRex)
+        .flatMap(rex -> getInputRefs(rex).stream())
+        .toList();
+  }
 }

core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java

@@ -227,6 +227,10 @@ public enum BuiltinFunctionName {
   NULLIF(FunctionName.of("nullif")),
   ISNULL(FunctionName.of("isnull")),
 
+  IS_PRESENT(FunctionName.of("ispresent")),
+  IS_EMPTY(FunctionName.of("isempty")),
+  IS_BLANK(FunctionName.of("isblank")),
+
   ROW_NUMBER(FunctionName.of("row_number")),
   RANK(FunctionName.of("rank")),
   DENSE_RANK(FunctionName.of("dense_rank")),

core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java

@@ -202,6 +202,7 @@ void populate() {
       registerOperator(IS_NULL, SqlStdOperatorTable.IS_NULL);
       registerOperator(IF, SqlStdOperatorTable.CASE);
       registerOperator(IFNULL, SqlStdOperatorTable.COALESCE);
+      registerOperator(IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL);
 
       // Register library operator
       registerOperator(REGEXP, SqlLibraryOperators.REGEXP);
@@ -371,6 +372,28 @@ void populate() {
                       builder.makeCall(SqlStdOperatorTable.EQUALS, arg1, arg2),
                       builder.makeNullLiteral(arg1.getType()),
                       arg1));
+      register(
+          IS_EMPTY,
+          ((FunctionImp1)
+              (builder, arg) ->
+                  builder.makeCall(
+                      SqlStdOperatorTable.OR,
+                      builder.makeCall(SqlStdOperatorTable.IS_NULL, arg),
+                      builder.makeCall(SqlStdOperatorTable.IS_EMPTY, arg))));
+      register(
+          IS_BLANK,
+          ((FunctionImp1)
+              (builder, arg) ->
+                  builder.makeCall(
+                      SqlStdOperatorTable.OR,
+                      builder.makeCall(SqlStdOperatorTable.IS_NULL, arg),
+                      builder.makeCall(
+                          SqlStdOperatorTable.IS_EMPTY,
+                          builder.makeCall(
+                              SqlStdOperatorTable.TRIM,
+                              builder.makeFlag(Flag.BOTH),
+                              builder.makeLiteral(" "),
+                              arg)))));
     }
   }
 

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -45,6 +45,8 @@ Argument type: all the supported data type.
 
 Return type: BOOLEAN
 
+Synonyms: `ISPRESENT`_
+
 Example::
 
     os> source=accounts | where not isnotnull(employer) | fields account_number, employer
@@ -238,3 +240,85 @@ Example::
     | Dale      | Adams    | 33  |
     +-----------+----------+-----+
 
+ISPRESENT
+---------
+
+Description
+>>>>>>>>>>>
+
+Version: 3.1.0
+
+Usage: ispresent(field) return true if the field exists.
+
+Argument type: all the supported data type.
+
+Return type: BOOLEAN
+
+Synonyms: `ISNOTNULL`_
+
+Example::
+
+    PPL> source=accounts | where ispresent(employer) | fields employer, firstname
+    fetched rows / total rows = 3/3
+    +----------+-----------+
+    | employer | firstname |
+    |----------+-----------|
+    | Pyrami   | Amber     |
+    | Netagy   | Hattie    |
+    | Quility  | Nanette   |
+    +----------+-----------+
+
+ISBLANK
+-------
+
+Description
+>>>>>>>>>>>
+
+Version: 3.1.0
+
+Usage: isblank(field) returns true if the field is null, an empty string, or contains only white space.
+
+Argument type: all the supported data type.
+
+Return type: BOOLEAN
+
+Example::
+
+    PPL> source=accounts | eval temp = ifnull(employer, '   ') | eval `isblank(employer)` = isblank(employer), `isblank(temp)` = isblank(temp) | fields `isblank(temp)`, temp, `isblank(employer)`, employer
+    fetched rows / total rows = 4/4
+    +---------------+---------+-------------------+----------+
+    | isblank(temp) | temp    | isblank(employer) | employer |
+    |---------------+---------+-------------------+----------|
+    | False         | Pyrami  | False             | Pyrami   |
+    | False         | Netagy  | False             | Netagy   |
+    | False         | Quility | False             | Quility  |
+    | True          |         | True              | null     |
+    +---------------+---------+-------------------+----------+
+
+
+ISEMPTY
+-------
+
+Description
+>>>>>>>>>>>
+
+Version: 3.1.0
+
+Usage: isempty(field) returns true if the field is null or is an empty string.
+
+Argument type: all the supported data type.
+
+Return type: BOOLEAN
+
+Example::
+
+    PPL> source=accounts | eval temp = ifnull(employer, '   ') | eval `isempty(employer)` = isempty(employer), `isempty(temp)` = isempty(temp) | fields `isempty(temp)`, temp, `isempty(employer)`, employer
+    fetched rows / total rows = 4/4
+    +---------------+---------+-------------------+----------+
+    | isempty(temp) | temp    | isempty(employer) | employer |
+    |---------------+---------+-------------------+----------|
+    | False         | Pyrami  | False             | Pyrami   |
+    | False         | Netagy  | False             | Netagy   |
+    | False         | Quility | False             | Quility  |
+    | False         |         | True              | null     |
+    +---------------+---------+-------------------+----------+