Merge remote-tracking branch 'upstream/main' into big5

Author: LantaoJin

.github/workflows/enforce-labels.yml

@@ -0,0 +1,13 @@
+name: Enforce PR labels
+
+on:
+  pull_request:
+    types: [labeled, unlabeled, opened, edited, synchronize]
+jobs:
+  enforce-label:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: yogevbd/enforce-label-action@2.1.0
+      with:
+        REQUIRED_LABELS_ANY: "breaking,feature,enhancement,bug,infrastructure,dependencies,documentation,maintenance,skip-changelog"
+        REQUIRED_LABELS_ANY_DESCRIPTION: "A release label is required: ['breaking', 'bug', 'dependencies', 'documentation', 'enhancement', 'feature', 'infrastructure', 'maintenance', 'skip-changelog']"

core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java

@@ -45,6 +45,7 @@ public enum SpanUnit {
   /** Util method to get span unit given the unit name. */
   public static SpanUnit of(String unit) {
     switch (unit) {
+      case null:
       case "":
         return NONE;
       case "M":

core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java

@@ -51,13 +51,14 @@ public SqlReturnTypeInference getReturnTypeInference() {
   public UDFOperandMetadata getOperandMetadata() {
     return UDFOperandMetadata.wrap(
         (CompositeOperandTypeChecker)
-            OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER, SqlTypeFamily.STRING)
+            OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)
                 .or(
                     OperandTypes.family(
-                        SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER, SqlTypeFamily.STRING))
+                        SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING))
+                // TODO: numeric span should support decimal as its interval
                 .or(
                     OperandTypes.family(
-                        SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.ANY)));
+                        SqlTypeFamily.NUMERIC, SqlTypeFamily.INTEGER, SqlTypeFamily.ANY)));
   }
 
   public static class SpanImplementor implements NotNullImplementor {

docs/dev/intro-v3-engine.md

@@ -1,4 +1,4 @@
-# PPL Engine V3 (for 3.0.0-beta)
+# PPL Engine V3 (for 3.0.0)
 
 ---
 ## 1. Motivations
@@ -24,7 +24,7 @@ Find more details in [V3 Architecture](./intro-v3-architecture.md).
 ---
 ## 2. What's New
 
-In the initial release of the V3 engine (3.0.0-beta), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior.
+In the initial release of the V3 engine (3.0.0), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior.
 
 * **[Join](../user/ppl/cmd/join.rst) Command**
 * **[Lookup](../user/ppl/cmd/lookup.rst) Command**
@@ -35,7 +35,7 @@ In the initial release of the V3 engine (3.0.0-beta), the main new features focu
 
 ### 3.1 Breaking Changes
 
-Because of implementation changed internally, following behaviors are changed from 3.0.0-beta. (Behaviors in V3 is correct)
+Because of implementation changed internally, following behaviors are changed from 3.0.0. (Behaviors in V3 is correct)
 
 |                       Item                       |    V2     |          V3          |
 |:------------------------------------------------:|:---------:|:--------------------:|
@@ -51,7 +51,7 @@ Because of implementation changed internally, following behaviors are changed fr
 
 ### 3.2 Fallback Mechanism
 
-As v3 engine is experimental in 3.0.0-beta, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...".
+As v3 engine is experimental in 3.0.0, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...".
 
 ### 3.3 Limitations
 
@@ -66,24 +66,12 @@ For the following functionalities in V3 engine, the query will be forwarded to t
 
 #### Unsupported functionalities
 - All SQL queries
-- `trendline`
-- `show datasource`
-- `explain`
-- `describe`
-- `top` and `rare`
-- `fillnull`
-- `patterns`
 - `dedup` with `consecutive=true`
 - Search relevant commands
   - AD
   - ML
   - Kmeans
 - Commands with `fetch_size` parameter
-- query with metadata fields, `_id`, `_doc`, etc.
-- Json relevant functions
-  - cast to json
-  - json
-  - json_valid
 - Search relevant functions
   - match
   - match_phrase
@@ -105,6 +93,5 @@ If you're interested in the new query engine, please find more details in [V3 Ar
 The following items are on our roadmap with high priority:
 - Resolve the [V3 limitation](#33-limitations).
 - Advancing pushdown optimization and benchmarking
-- Backport to 2.19.x
 - Unified the PPL syntax between [PPL-on-OpenSearch](https://github.com/opensearch-project/sql/blob/main/ppl/src/main/antlr/OpenSearchPPLParser.g4) and [PPL-on-Spark](https://github.com/opensearch-project/opensearch-spark/blob/main/ppl-spark-integration/src/main/antlr4/OpenSearchPPLParser.g4)
 - Support more DSL aggregation

docs/user/limitations/limitations.rst

@@ -130,3 +130,31 @@ The response in JSON format is::
       },
       "status": 400
     }
+
+Limitations on Calcite Engine
+=============================
+
+Since 3.0.0, we introduce Apache Calcite as an experimental query engine. Please see `introduce v3 engine <../../../dev/intro-v3-engine.md>`_.
+For the following functionalities, the query will be forwarded to the V2 query engine.
+
+* All SQL queries
+
+* ``dedup`` with ``consecutive=true``
+
+* Search relevant commands
+
+    * AD
+    * ML
+    * Kmeans
+
+* Commands with ``fetch_size`` parameter
+
+* Search relevant functions
+
+    * match
+    * match_phrase
+    * match_bool_prefix
+    * match_phrase_prefix
+    * simple_query_string
+    * query_string
+    * multi_match

docs/user/ppl/limitations/limitations.rst

@@ -84,3 +84,31 @@ plugins.query.field_type_tolerance setting is enabled, the SQL/PPL plugin will h
 scalar data types, allowing basic queries (e.g., source = tbl | where condition). However, using multi-value
 fields in expressions or functions will result in exceptions. If this setting is disabled or absent, only the
 first element of an array is returned, preserving the default behavior.
+
+Unsupported Functionalities in Calcite Engine
+=============================================
+
+Since 3.0.0, we introduce Apache Calcite as an experimental query engine. Please see `introduce v3 engine <../../../dev/intro-v3-engine.md>`_.
+For the following functionalities, the query will be forwarded to the V2 query engine.
+
+* All SQL queries
+
+* ``dedup`` with ``consecutive=true``
+
+* Search relevant commands
+
+    * AD
+    * ML
+    * Kmeans
+
+* Commands with ``fetch_size`` parameter
+
+* Search relevant functions
+
+    * match
+    * match_phrase
+    * match_bool_prefix
+    * match_phrase_prefix
+    * simple_query_string
+    * query_string
+    * multi_match

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -20,4 +20,9 @@ public void init() throws Exception {
   @Override
   @Ignore("test only in v2")
   public void testExplainModeUnsupportedInV2() throws IOException {}
+
+  @Override
+  public void testExplain() throws IOException {
+    super.testExplain();
+  }
 }

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java

@@ -501,7 +501,13 @@ public void testCountBySpanForCustomFormats() throws IOException {
         actual,
         schema("timestamp_span", "timestamp"),
         schema("count(custom_no_delimiter_ts)", "bigint"));
-    verifyDataRows(actual, rows(1, "1961-04-12 10:00:00"), rows(1, "1984-10-20 15:00:00"));
+    // TODO: Span has different behavior between pushdown and non-pushdown for timestamp before
+    // 1971. V2 engine will have the same issue.
+    // https://github.com/opensearch-project/sql/issues/3827
+    verifyDataRows(
+        actual,
+        rows(1, isPushdownEnabled() ? "1961-04-12 09:00:00" : "1961-04-12 10:00:00"),
+        rows(1, "1984-10-20 15:00:00"));
 
     actual =
         executeQuery(

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLSortIT.java

@@ -217,4 +217,22 @@ public void testSortWithNullValue() throws IOException {
         rows("Elinor", null),
         rows("Virginia", null));
   }
+
+  @Test
+  public void testSortDate() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | sort birthdate | fields firstname, birthdate", TEST_INDEX_BANK));
+    verifySchema(result, schema("firstname", "string"), schema("birthdate", "timestamp"));
+    verifyDataRowsInOrder(
+        result,
+        rows("Amber JOHnny", "2017-10-23 00:00:00"),
+        rows("Hattie", "2017-11-20 00:00:00"),
+        rows("Nanette", "2018-06-23 00:00:00"),
+        rows("Elinor", "2018-06-27 00:00:00"),
+        rows("Dillard", "2018-08-11 00:00:00"),
+        rows("Virginia", "2018-08-19 00:00:00"),
+        rows("Dale", "2018-11-13 23:33:20"));
+  }
 }

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java

@@ -5,6 +5,13 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyOrder;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Test;
 import org.opensearch.sql.ppl.SortCommandIT;
 
 public class CalciteSortCommandIT extends SortCommandIT {
@@ -14,4 +21,12 @@ public void init() throws Exception {
     enableCalcite();
     disallowCalciteFallback();
   }
+
+  // TODO: Move this test to SortCommandIT once head-then-sort is fixed in v2.
+  @Test
+  public void testHeadThenSort() throws IOException {
+    JSONObject result =
+        executeQuery(String.format("source=%s | head 2 | sort age | fields age", TEST_INDEX_BANK));
+    verifyOrder(result, rows(32), rows(36));
+  }
 }