Merge remote-tracking branch 'origin/main' into addAppendpipe

Author: xinyual

.github/workflows/maven-publish-modules.yml

@@ -0,0 +1,47 @@
+name: Publish unified query modules to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
+
+env:
+  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
+
+jobs:
+  publish-unified-query-modules:
+    strategy:
+      fail-fast: false
+    if: github.repository == 'opensearch-project/sql'
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - uses: actions/checkout@v3
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
+        with:
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          ./gradlew publishUnifiedQueryPublicationToSnapshotsRepository

.github/workflows/maven-publish.yml

@@ -8,9 +8,6 @@ on:
       - '[0-9]+.[0-9]+'
       - '[0-9]+.x'
 
-env:
-  SNAPSHOT_REPO_URL: https://ci.opensearch.org/ci/dbc/snapshots/maven/
-
 jobs:
   build-and-publish-snapshots:
     strategy:

.github/workflows/publish-async-query-core.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'async-query-core/**'
       - '.github/workflows/publish-async-query-core.yml'
@@ -18,7 +18,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-async-query-core.json
 
 jobs:
@@ -47,8 +46,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set commit ID
         id: set_commit
@@ -83,4 +93,4 @@ jobs:
           source ./.github/maven-publish-utils.sh
           
           # Call the main function for async-query-core
-          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"
+          publish_async_query_core "${{ steps.extract_version.outputs.VERSION }}" "${{ steps.set_commit.outputs.commit_id }}"

.github/workflows/publish-grammar-files.yml

@@ -5,8 +5,8 @@ on:
   push:
     branches:
       - main
-      - 1.*
-      - 2.*
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.x'
     paths:
       - 'language-grammar/src/main/antlr4/**'
       - 'language-grammar/build.gradle'
@@ -19,7 +19,6 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  SNAPSHOT_REPO_URL: https://central.sonatype.com/repository/maven-snapshots/
   COMMIT_MAP_FILENAME: commit-history-language-grammar.json
 
 jobs:
@@ -51,8 +50,19 @@ jobs:
           export-env: true
         env:
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
-          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
-          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
+          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
+
+      - name: Export SNAPSHOT_REPO_URL
+        run: |
+          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
+          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
+          aws-region: us-east-1
 
       - name: Set version
         id: set_version

.github/workflows/sql-test-and-build-workflow.yml

@@ -179,7 +179,7 @@ jobs:
             doctest/build/testclusters/docTestCluster-0/logs/*
             integ-test/build/testclusters/*/logs/*
 
-  bwc-tests:
+  bwc-tests-rolling-upgrade:
     needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
     strategy:
@@ -190,37 +190,83 @@ jobs:
       options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - name: Run start commands
-      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
-
-    - uses: actions/checkout@v4
-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v4
-      with:
-        distribution: 'temurin'
-        java-version: ${{ matrix.java }}
-
-    - name: Run backward compatibility tests
-      run: |
-        chown -R 1000:1000 `pwd`
-        su `id -un 1000` -c "./scripts/bwctest.sh"
-
-    - name: Upload test reports
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v4
-      continue-on-error: true
-      with:
-        name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
-        path: |
-          sql/build/reports/**
-          ppl/build/reports/**
-          core/build/reports/**
-          common/build/reports/**
-          opensearch/build/reports/**
-          integ-test/build/reports/**
-          protocol/build/reports/**
-          legacy/build/reports/**
-          plugin/build/reports/**
-          doctest/build/testclusters/docTestCluster-0/logs/*
-          integ-test/build/testclusters/*/logs/*
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in rolling upgrade
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-rolling-upgrade.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*
+
+  bwc-tests-full-restart:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21, 24]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      - name: Run backward compatibility tests in full restart
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./scripts/bwctest-full-restart.sh"
+
+      - name: Upload test reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+          path: |
+            sql/build/reports/**
+            ppl/build/reports/**
+            core/build/reports/**
+            common/build/reports/**
+            opensearch/build/reports/**
+            integ-test/build/reports/**
+            protocol/build/reports/**
+            legacy/build/reports/**
+            plugin/build/reports/**
+            doctest/build/testclusters/docTestCluster-0/logs/*
+            integ-test/build/testclusters/*/logs/*

DEVELOPER_GUIDE.rst

@@ -173,7 +173,6 @@ Here are other files and sub-folders that you are likely to touch:
 - ``build.gradle``: Gradle build script.
 - ``docs``: documentation for developers and reference manual for users.
 - ``doc-test``: code that run .rst docs in ``docs`` folder by Python doctest library.
-- ``language-grammar``: centralized package for ANTLR grammar files. See `Language Grammar Package`_ for details.
 
 Note that other related project code has already merged into this single repository together:
 
@@ -444,29 +443,3 @@ with an appropriate label `backport <backport-branch-name>` is merged to main wi
 PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the
 backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR
 to the `1.x` branch.
-
-Language Grammar Package
-========================
-
-The ``language-grammar`` package serves as a centralized repository for all ANTLR grammar files used throughout the OpenSearch SQL project. This package contains the definitive versions of grammar files for:
-
-- SQL parsing (``OpenSearchSQLParser.g4``, ``OpenSearchSQLLexer.g4``)
-- PPL parsing (``OpenSearchPPLParser.g4``, ``OpenSearchPPLLexer.g4``)
-- Legacy SQL parsing (``OpenSearchLegacySqlParser.g4``, ``OpenSearchLegacySqlLexer.g4``)
-- Spark SQL extensions (``SparkSqlBase.g4``, ``FlintSparkSqlExtensions.g4``, ``SqlBaseParser.g4``, ``SqlBaseLexer.g4``)
-
-Purpose
--------
-
-The language-grammar package enables sharing of grammar files between the main SQL repository and the Spark repository, ensuring consistency and reducing duplication. Once updated, the package automatically triggers CI to upload the new version to Maven Central for consumption by other projects.
-
-Updating Grammar Files
-----------------------
-
-When grammar files are modified in their respective modules (``sql/``, ``ppl/``, ``legacy/``, ``async-query-core/``), they must be manually copied to the ``language-grammar/src/main/antlr4/`` directory.
-
-**Workflow:**
-
-1. Modify grammar files in their source locations (e.g., ``sql/src/main/antlr/``)
-2. Copy updated files to ``language-grammar/src/main/antlr4/``
-3. Commit changes to trigger automatic Maven publication via CI

api/README.md

@@ -0,0 +1,75 @@
+# Unified Query API
+
+This module provides a high-level integration layer for the Calcite-based query engine, enabling external systems such as Apache Spark or command-line tools to parse and analyze queries without exposing low-level internals.
+
+## Overview
+
+The `UnifiedQueryPlanner` serves as the primary entry point for external consumers. It accepts PPL (Piped Processing Language) queries and returns Calcite `RelNode` logical plans as intermediate representation.
+
+## Usage
+
+Use the declarative, fluent builder API to initialize the `UnifiedQueryPlanner`.
+
+```java
+UnifiedQueryPlanner planner = UnifiedQueryPlanner.builder()
+    .language(QueryType.PPL)
+    .catalog("opensearch", schema)
+    .defaultNamespace("opensearch")
+    .cacheMetadata(true)
+    .build();
+
+RelNode plan = planner.plan("source = opensearch.test");
+```
+
+## Development & Testing
+
+A set of unit tests is provided to validate planner behavior.
+
+To run tests:
+
+```
+./gradlew :api:test
+```
+
+## Integration Guide
+
+This guide walks through how to integrate unified query planner into your application.
+
+### Step 1: Add Dependency
+
+The module is currently published as a snapshot to the AWS Sonatype Snapshots repository. To include it as a dependency in your project, add the following to your `pom.xml` or `build.gradle`:
+
+```xml
+<dependency>
+  <groupId>org.opensearch.query</groupId>
+  <artifactId>unified-query-api</artifactId>
+  <version>YOUR_VERSION_HERE</version>
+</dependency>
+```
+
+### Step 2: Implement a Calcite Schema
+
+You must implement the Calcite `Schema` interface and register them using the fluent `catalog()` method on the builder.
+
+```java
+public class MySchema extends AbstractSchema {
+  @Override
+  protected Map<String, Table> getTableMap() {
+    return Map.of(
+      "test_table",
+      new AbstractTable() {
+        @Override
+        public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+          return typeFactory.createStructType(
+            List.of(typeFactory.createSqlType(SqlTypeName.INTEGER)),
+            List.of("id"));
+        }
+      });
+  }
+}
+```
+
+## Future Work
+
+- Expand support to SQL language.
+- Extend planner to generate optimized physical plans using Calcite's optimization frameworks.