Merge remote-tracking branch 'origin/main' into pushdown-ip

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.calcite.plan;
+
+import java.util.Collections;
+import java.util.List;
+import lombok.Getter;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.hint.RelHint;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rex.RexNode;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+/** System level limit logical plan, comparing to user level plan {@link LogicalSort}. */
+public class LogicalSystemLimit extends Sort {
+
+  public enum SystemLimitType {
+    /**
+     * System limit type for system level limit.
+     *
+     * <p>This type is used to indicate that the limit is applied to the system level.
+     */
+    QUERY_SIZE_LIMIT
+  }
+
+  @Getter private final SystemLimitType type;
+
+  private LogicalSystemLimit(
+      SystemLimitType type,
+      RelOptCluster cluster,
+      RelTraitSet traitSet,
+      RelNode input,
+      RelCollation collation,
+      @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
+    this(type, cluster, traitSet, Collections.emptyList(), input, collation, offset, fetch);
+  }
+
+  private LogicalSystemLimit(
+      SystemLimitType type,
+      RelOptCluster cluster,
+      RelTraitSet traitSet,
+      List<RelHint> hints,
+      RelNode input,
+      RelCollation collation,
+      @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
+    super(cluster, traitSet, hints, input, collation, offset, fetch);
+    assert traitSet.containsIfApplicable(Convention.NONE);
+    this.type = type;
+  }
+
+  public static LogicalSystemLimit create(SystemLimitType type, RelNode input, RexNode fetch) {
+    return create(type, input, input.getTraitSet().getCollation(), null, fetch);
+  }
+
+  public static LogicalSystemLimit create(
+      SystemLimitType type,
+      RelNode input,
+      RelCollation collation,
+      @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
+    RelOptCluster cluster = input.getCluster();
+    collation = RelCollationTraitDef.INSTANCE.canonize(collation);
+    RelTraitSet traitSet = input.getTraitSet().replace(Convention.NONE).replace(collation);
+    return new LogicalSystemLimit(type, cluster, traitSet, input, collation, offset, fetch);
+  }
+
+  @Override
+  public Sort copy(
+      RelTraitSet traitSet,
+      RelNode newInput,
+      RelCollation newCollation,
+      @Nullable RexNode offset,
+      @Nullable RexNode fetch) {
+    return new LogicalSystemLimit(
+        this.type, getCluster(), traitSet, hints, newInput, newCollation, offset, fetch);
+  }
+
+  @Override
+  public RelWriter explainTerms(RelWriter pw) {
+    super.explainTerms(pw);
+    // Show type in the explain
+    pw.item("type", type);
+    return pw;
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/plan/Scannable.java

@@ -16,5 +16,5 @@
  */
 public interface Scannable {
 
-  public Enumerable<@Nullable Object> scanWithLimit();
+  public Enumerable<@Nullable Object> scan();
 }

core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java

@@ -302,7 +302,7 @@ protected PreparedResult implement(RelRoot root) {
       RelDataType resultType = root.rel.getRowType();
       boolean isDml = root.kind.belongsTo(SqlKind.DML);
       if (root.rel instanceof Scannable scannable) {
-        final Bindable bindable = dataContext -> scannable.scanWithLimit();
+        final Bindable bindable = dataContext -> scannable.scan();
 
         return new PreparedResultImpl(
             resultType,

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -36,6 +36,8 @@
 import org.opensearch.sql.calcite.CalcitePlanContext;
 import org.opensearch.sql.calcite.CalciteRelNodeVisitor;
 import org.opensearch.sql.calcite.OpenSearchSchema;
+import org.opensearch.sql.calcite.plan.LogicalSystemLimit;
+import org.opensearch.sql.calcite.plan.LogicalSystemLimit.SystemLimitType;
 import org.opensearch.sql.common.response.ResponseListener;
 import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.common.setting.Settings.Key;
@@ -100,7 +102,7 @@ public void executeWithCalcite(
                     CalcitePlanContext.create(
                         buildFrameworkConfig(), getQuerySizeLimit(), queryType);
                 RelNode relNode = analyze(plan, context);
-                RelNode optimized = optimize(relNode);
+                RelNode optimized = optimize(relNode, context);
                 RelNode calcitePlan = convertToCalcitePlan(optimized);
                 executionEngine.execute(calcitePlan, context, listener);
                 return null;
@@ -133,7 +135,7 @@ public void explainWithCalcite(
                     CalcitePlanContext.create(
                         buildFrameworkConfig(), getQuerySizeLimit(), queryType);
                 RelNode relNode = analyze(plan, context);
-                RelNode optimized = optimize(relNode);
+                RelNode optimized = optimize(relNode, context);
                 RelNode calcitePlan = convertToCalcitePlan(optimized);
                 executionEngine.explain(calcitePlan, format, context, listener);
                 return null;
@@ -250,8 +252,13 @@ public PhysicalPlan plan(LogicalPlan plan) {
     return planner.plan(plan);
   }
 
-  public RelNode optimize(RelNode plan) {
-    return planner.customOptimize(plan);
+  /**
+   * Try to optimize the plan by appending a limit operator for QUERY_SIZE_LIMIT Don't add for
+   * `EXPLAIN` to avoid changing its output plan.
+   */
+  public RelNode optimize(RelNode plan, CalcitePlanContext context) {
+    return LogicalSystemLimit.create(
+        SystemLimitType.QUERY_SIZE_LIMIT, plan, context.relBuilder.literal(context.querySizeLimit));
   }
 
   private boolean isCalciteFallbackAllowed() {

docs/user/ppl/cmd/expand.rst

@@ -11,7 +11,7 @@ expand
 
 Description
 ============
-     (From 3.1.0)
+| (Experimental)
 
 Use the ``expand`` command on a nested array field to transform a single
 document into multiple documents—each containing one element from the array.

docs/user/ppl/index.rst

@@ -40,59 +40,75 @@ The query start with search command and then flowing a set of command delimited
 
   - `Cross-Cluster Search <admin/cross_cluster_search.rst>`_
 
+* **Language Structure**
+
+  - `Identifiers <general/identifiers.rst>`_
+
+  - `Data Types <general/datatypes.rst>`_
+
 * **Commands**
 
   - `Syntax <cmd/syntax.rst>`_
 
   - `ad command <cmd/ad.rst>`_
 
+  - `appendcol command <cmd/appendcol.rst>`_
+
   - `dedup command <cmd/dedup.rst>`_
 
   - `describe command <cmd/describe.rst>`_
 
-  - `show datasources command <cmd/showdatasources.rst>`_
-
   - `eval command <cmd/eval.rst>`_
 
+  - `eventstats command <cmd/eventstats.rst>`_
+
+  - `expand command <cmd/expand.rst>`_
+
+  - `explain command <cmd/explain.rst>`_
+
   - `fields command <cmd/fields.rst>`_
 
+  - `fillnull command <cmd/fillnull.rst>`_
+
+  - `flatten command  <cmd/flatten.rst>`_
+
   - `grok command <cmd/grok.rst>`_
 
+  - `head command <cmd/head.rst>`_
+  
+  - `join command  <cmd/join.rst>`_
+
   - `kmeans command <cmd/kmeans.rst>`_
 
+  - `lookup command <cmd/lookup.rst>`_
+
+  - `metadata commands <cmd/information_schema.rst>`_
+
   - `ml command <cmd/ml.rst>`_
 
   - `parse command <cmd/parse.rst>`_
 
   - `patterns command <cmd/patterns.rst>`_
 
+  - `rare command <cmd/rare.rst>`_
+
   - `rename command <cmd/rename.rst>`_
 
   - `search command <cmd/search.rst>`_
 
+  - `show datasources command <cmd/showdatasources.rst>`_
+
   - `sort command <cmd/sort.rst>`_
 
   - `stats command <cmd/stats.rst>`_
 
-  - `trendline command <cmd/trendline.rst>`_
-
-  - `where command <cmd/where.rst>`_
-
-  - `head command <cmd/head.rst>`_
-  
-  - `rare command <cmd/rare.rst>`_
+  - `subquery (aka subsearch) command <cmd/subquery.rst>`_
 
   - `top command <cmd/top.rst>`_
 
-  - `metadata commands <cmd/information_schema.rst>`_
-
-  - `(Experimental)(From 3.0.0) join command <cmd/join.rst>`_
-
-  - `(Experimental)(From 3.0.0) lookup command <cmd/lookup.rst>`_
-
-  - `(Experimental)(From 3.0.0) subquery (aka subsearch) command <cmd/subquery.rst>`_
+  - `trendline command <cmd/trendline.rst>`_
 
-  - `(Experimental)(From 3.1.0) eventstats command <cmd/eventstats.rst>`_
+  - `where command <cmd/where.rst>`_
 
 * **Functions**
 
@@ -124,12 +140,6 @@ The query start with search command and then flowing a set of command delimited
 
   - `Optimization <../../user/optimization/optimization.rst>`_
 
-* **Language Structure**
-
-  - `Identifiers <general/identifiers.rst>`_
-
-  - `Data Types <general/datatypes.rst>`_
-
 * **Limitations**
 
   - `Limitations <limitations/limitations.rst>`_

docs/user/ppl/limitations/limitations.rst

@@ -102,13 +102,3 @@ For the following functionalities, the query will be forwarded to the V2 query e
     * Kmeans
 
 * Commands with ``fetch_size`` parameter
-
-* Search relevant functions
-
-    * match
-    * match_phrase
-    * match_bool_prefix
-    * match_phrase_prefix
-    * simple_query_string
-    * query_string
-    * multi_match

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java

@@ -80,7 +80,8 @@ public void testExplainCommandCost() throws IOException {
             ? loadFromFile("expectedOutput/calcite/explain_filter_cost_w_pushdown.txt")
             : loadFromFile("expectedOutput/calcite/explain_filter_cost_wo_pushdown.txt");
     assertTrue(
-        String.format("Got: %s\n, expected: %s", result, expected), result.contains(expected));
+        String.format("Got: %s\n, expected: %s", result, expected),
+        result.contains(expected.trim()));
   }
 
   @Test