Merge remote-tracking branch 'origin/main' into issues/4580

Author: yuancu

.github/workflows/maven-publish.yml

@@ -8,9 +8,6 @@ on:
       - '[0-9]+.[0-9]+'
       - '[0-9]+.x'
 
-env:
-  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
-
 jobs:
   build-and-publish-snapshots:
     strategy:

.github/workflows/publish-async-query-core.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'async-query-core/**'
       - '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-async-query-core.json
 
 jobs:
@@ -47,8 +46,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set commit ID
         id: set_commit
@@ -83,4 +93,4 @@ jobs:
           source ./.github/maven-publish-utils.sh
           
           # Call the main function for async-query-core
-          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
+          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

.github/workflows/publish-grammar-files.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'language-grammar/src/main/antlr4/**'
       - 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-language-grammar.json
 
 jobs:
@@ -51,8 +50,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set version
         id: set_version

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -80,6 +80,7 @@
 import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.ast.tree.Values;
+import org.opensearch.sql.calcite.plan.OpenSearchConstants;
 
 /** Class of static methods to create specific node instances. */
 @UtilityClass
@@ -491,15 +492,19 @@ public static Span spanFromSpanLengthLiteral(
       UnresolvedExpression field, Literal spanLengthLiteral) {
     if (spanLengthLiteral.getType() == DataType.STRING) {
       String spanText = spanLengthLiteral.getValue().toString();
-      String valueStr = spanText.replaceAll("[^0-9]", "");
-      String unitStr = spanText.replaceAll("[0-9]", "");
+      String valueStr = spanText.replaceAll("[^0-9-]", "");
+      String unitStr = spanText.replaceAll("[0-9-]", "");
 
       if (valueStr.isEmpty()) {
         // No numeric value found, use the literal as-is
         return new Span(field, spanLengthLiteral, SpanUnit.NONE);
       } else {
         // Parse numeric value and unit
         Integer value = Integer.parseInt(valueStr);
+        if (value <= 0) {
+          throw new IllegalArgumentException(
+              String.format("Zero or negative time interval not supported: %s", spanText));
+        }
         SpanUnit unit = unitStr.isEmpty() ? SpanUnit.NONE : SpanUnit.of(unitStr);
         return span(field, intLiteral(value), unit);
       }
@@ -713,4 +718,9 @@ public static Bin bin(UnresolvedExpression field, Argument... arguments) {
       return DefaultBin.builder().field(field).alias(alias).build();
     }
   }
+
+  /** Get a reference to the implicit timestamp field {@code @timestamp} */
+  public static Field referImplicitTimestampField() {
+    return AstDSL.field(OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP);
+  }
 }

core/src/main/java/org/opensearch/sql/ast/statement/Explain.java

@@ -39,7 +39,9 @@ public enum ExplainFormat {
     SIMPLE,
     STANDARD,
     EXTENDED,
-    COST
+    COST,
+    /** Formats explain output in yaml format. */
+    YAML
   }
 
   public static ExplainFormat format(String format) {

core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java

@@ -109,6 +109,22 @@ public boolean equals(Object o) {
     public int hashCode() {
       return Objects.hash(root, calcite);
     }
+
+    public static ExplainResponse normalizeLf(ExplainResponse response) {
+      ExecutionEngine.ExplainResponseNodeV2 calcite = response.getCalcite();
+      if (calcite != null) {
+        return new ExplainResponse(
+            new ExecutionEngine.ExplainResponseNodeV2(
+                normalizeLf(calcite.getLogical()),
+                normalizeLf(calcite.getPhysical()),
+                normalizeLf(calcite.getExtended())));
+      }
+      return response;
+    }
+
+    private static String normalizeLf(String value) {
+      return value == null ? null : value.replace("\r\n", "\n");
+    }
   }
 
   @AllArgsConstructor

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -201,7 +201,8 @@ public void explainWithLegacy(
       Explain.ExplainFormat format,
       Optional<Throwable> calciteFailure) {
     try {
-      if (format != null && format != Explain.ExplainFormat.STANDARD) {
+      if (format != null
+          && (format != Explain.ExplainFormat.STANDARD && format != Explain.ExplainFormat.YAML)) {
         throw new UnsupportedOperationException(
             "Explain mode " + format.name() + " is not supported in v2 engine");
       }

core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java

@@ -5,8 +5,10 @@
 
 package org.opensearch.sql.utils;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
 
@@ -21,11 +23,16 @@ public class YamlFormatter {
   static {
     YAMLFactory yamlFactory = new YAMLFactory();
     yamlFactory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER);
+    yamlFactory.enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS);
+    yamlFactory.enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE);
     yamlFactory.enable(YAMLGenerator.Feature.MINIMIZE_QUOTES); // Enable smart quoting
     yamlFactory.enable(
         YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings
     yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR);
     YAML_MAPPER = new ObjectMapper(yamlFactory);
+
+    YAML_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    YAML_MAPPER.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false);
   }
 
   /**

docs/category.json

@@ -1,11 +1,11 @@
 {
   "bash": [
-    "user/ppl/interfaces/endpoint.rst",
-    "user/ppl/interfaces/protocol.rst",
     "user/optimization/optimization.rst",
     "user/admin/settings.rst"
   ],
-  "ppl_cli": [
+  "bash_calcite": [
+    "user/ppl/interfaces/endpoint.rst",
+    "user/ppl/interfaces/protocol.rst"
   ],
   "sql_cli": [
     "user/dql/expressions.rst",

docs/user/ppl/cmd/stats.rst

@@ -58,8 +58,8 @@ stats [bucket_nullable=bool] <aggregation>... [by-clause]
 
 * span-expression: optional, at most one.
 
- * Syntax: span(field_expr, interval_expr)
- * Description: The unit of the interval expression is the natural unit by default. **If the field is a date/time type field, the aggregation results always ignore null bucket**. And the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``.
+ * Syntax: span([field_expr,] interval_expr)
+ * Description: The unit of the interval expression is the natural unit by default. If ``field_expr`` is omitted, span will use the implicit ``@timestamp`` field. An error will be thrown if this field doesn't exist. **If the field is a date/time type field, the aggregation results always ignore null bucket**. And the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``.
 * Available time unit:
 
 +----------------------------+
@@ -580,7 +580,7 @@ Description
 
 Version: 3.3.0 (Calcite engine only)
 
-Usage: LIST(expr). Collects all values from the specified expression into an array. Values are converted to strings, nulls are filtered, and duplicates are preserved. 
+Usage: LIST(expr). Collects all values from the specified expression into an array. Values are converted to strings, nulls are filtered, and duplicates are preserved.
 The function returns up to 100 values with no guaranteed ordering.
 
 * expr: The field expression to collect values from.
@@ -977,3 +977,18 @@ PPL query::
     | 1   | 2025-01-01 | 2      |
     +-----+------------+--------+
 
+
+Example 18: Calculate the count by the implicit @timestamp field
+================================================================
+
+This example demonstrates that if you omit the field parameter in the span function, it will automatically use the implicit ``@timestamp`` field.
+
+PPL query::
+
+    PPL> source=big5 | stats count() by span(1month)
+    fetched rows / total rows = 1/1
+    +---------+---------------------+
+    | count() | span(1month)        |
+    |---------+---------------------|
+    | 1       | 2023-01-01 00:00:00 |
+    +---------+---------------------+