Merge remote-tracking branch 'origin/main' into issues/4136

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,7 +10,7 @@ Resolves #[Issue number to be closed when this PR is merged]
 - [ ] New functionality has been documented.
  - [ ] New functionality has javadoc added.
  - [ ] New functionality has a user manual doc added.
-- [ ] New PPL command [checklist](https://github.com/opensearch-project/sql/blob/main/DEVELOPER_GUIDE.rst#new-ppl-command-checklist) all confirmed.
+- [ ] New PPL command [checklist](https://github.com/opensearch-project/sql/blob/main/docs/dev/ppl-commands.md) all confirmed.
 - [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
 - [ ] Commits are signed per the DCO using `--signoff` or `-s`.
 - [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).

.github/maven-publish-utils.sh

@@ -4,6 +4,10 @@
 
 set -e
 
+# Flag to disable commit mapping functionality
+# Set to "true" to disable commit mapping operations
+DISABLE_COMMIT_MAPPING="${DISABLE_COMMIT_MAPPING:-true}"
+
 # Function to execute curl commands with retry and error handling
 execute_curl_with_retry() {
   local url="$1"
@@ -111,6 +115,11 @@ update_version_metadata() {
   local commit_id="$3"
   local snapshot_repo_url="${4:-$SNAPSHOT_REPO_URL}"
 
+  if [ "$DISABLE_COMMIT_MAPPING" = "true" ]; then
+    echo "Skipping version metadata update (commit mapping disabled)"
+    return 0
+  fi
+
   echo "Updating version metadata for ${artifact_id} version ${version} with commit ID ${commit_id}"
 
   TEMP_DIR=$(mktemp -d)
@@ -204,6 +213,11 @@ update_commit_mapping() {
   local commit_map_filename="${5:-$COMMIT_MAP_FILENAME}"
   local snapshot_repo_url="${6:-$SNAPSHOT_REPO_URL}"
 
+  if [ "$DISABLE_COMMIT_MAPPING" = "true" ]; then
+    echo "Skipping commit-version mapping update (commit mapping disabled)"
+    return 0
+  fi
+
   echo "Updating commit-version mapping for ${artifact_id}"
 
   # Create temp directory for work

.github/workflows/link-checker.yml

@@ -18,7 +18,7 @@ jobs:
         id: lychee
         uses: lycheeverse/lychee-action@master
         with:
-          args: --accept=200,403,429,999  "./**/*.html" "./**/*.md" "./**/*.txt" --exclude "https://aws.oss.sonatype.*|https://central.sonatype.*|http://localhost.*|https://localhost|https://odfe-node1:9200/|https://community.tableau.com/docs/DOC-17978|.*family.zzz|opensearch*|.*@amazon.com|.*email.com|.*@github.com|http://timestamp.verisign.com/scripts/timstamp.dll"
+          args: --accept=200,403,429,999  "./**/*.html" "./**/*.md" "./**/*.txt" --exclude "https://aws.oss.sonatype.*|https://ci.opensearch.*|https://central.sonatype.*|http://localhost.*|https://localhost|https://odfe-node1:9200/|https://community.tableau.com/docs/DOC-17978|.*family.zzz|opensearch*|.*@amazon.com|.*email.com|.*@github.com|http://timestamp.verisign.com/scripts/timstamp.dll"
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       - name: Fail if there were link errors

DEVELOPER_GUIDE.rst

@@ -17,15 +17,15 @@ Prerequisites
 JDK
 ---
 
-OpenSearch builds using Java 11 at a minimum and supports JDK 11, 14 and 17. This means you must have a JDK of supported version installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation::
+OpenSearch SQL plugin requires Java 21 for development and runtime. This means you must have JDK 21 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK installation::
 
    $ echo $JAVA_HOME
-   /Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home
+   /Library/Java/JavaVirtualMachines/jdk-21.jdk/Contents/Home
 
    $ java -version
-    openjdk version "11.0.1" 2018-10-16
-    OpenJDK Runtime Environment 18.9 (build 11.0.1+13)
-    OpenJDK 64-Bit Server VM 18.9 (build 11.0.1+13, mixed mode)
+   openjdk version "21.0.8" 2024-07-16 LTS
+   OpenJDK Runtime Environment (build 21.0.8+13-LTS)
+   OpenJDK 64-Bit Server VM (build 21.0.8+13-LTS, mixed mode, sharing)
 
 Here are the official instructions on how to set ``JAVA_HOME`` for different platforms: https://docs.oracle.com/cd/E19182-01/820-7851/inst_cli_jdk_javahome_t/. 
 
@@ -78,12 +78,12 @@ You can develop the plugin in your favorite IDEs such as Eclipse and IntelliJ ID
 Java Language Level
 -------------------
 
-Although later version of JDK is required to build the plugin, the Java language level needs to be Java 8 for compatibility. Only in this case your plugin works with OpenSearch running against JDK 8. Otherwise it will raise runtime exception when executing new API from new JDK. In case your IDE doesn’t set it right, you may want to double check your project setting after import.
+The plugin requires Java 21 for both development and runtime. Make sure your IDE is configured to use Java 21 as the project SDK and language level. In case your IDE doesn't set it right, you may want to double check your project setting after import.
 
 Remote Debugging
 ----------------
 
-Firstly you need to add the following configuration to the JVM used by your IDE. For Intellij IDEA, it should be added to ``<OpenSearch installation>/config/jvm.options`` file. After configuring this, an agent in JVM will listen on the port when your OpenSearch bootstraps and wait for IDE debugger to connect. So you should be able to debug by setting up a “Remote Run/Debug Configuration”::
+Firstly you need to add the following configuration to the JVM used by your IDE. For Intellij IDEA, it should be added to ``<OpenSearch installation>/config/jvm.options`` file. After configuring this, an agent in JVM will listen on the port when your OpenSearch bootstraps and wait for IDE debugger to connect. So you should be able to debug by setting up a "Remote Run/Debug Configuration"::
 
    -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005
 
@@ -94,26 +94,20 @@ running.
 
    ./gradlew opensearch-sql:run -DdebugJVM
 
-To connect to the cluster with the debugger in an IDE, you’ll need to
+To connect to the cluster with the debugger in an IDE, you'll need to
 connect to that port. For IntelliJ, see `attaching to a remote process <https://www.jetbrains.com/help/idea/attach-to-process.html#attach-to-remote>`_.
 
 License Header
 --------------
 
-Because our code is licensed under Apache 2, you need to add the following license header to all new source code files. To automate this whenever creating new file, you can follow instructions for your IDE::
-
-   /*
-    * Licensed under the Apache License, Version 2.0 (the "License").
-    * You may not use this file except in compliance with the License.
-    * A copy of the License is located at
-    * 
-    *    http://www.apache.org/licenses/LICENSE-2.0
-    * 
-    * or in the "license" file accompanying this file. This file is distributed 
-    * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
-    * express or implied. See the License for the specific language governing 
-    * permissions and limitations under the License.
-    */
+Because our code is licensed under Apache 2, you need to add the following license header to all new source code files. To automate this whenever creating new file, you can follow instructions for your IDE.
+
+.. code:: java
+
+  /*
+   * Copyright OpenSearch Contributors
+   * SPDX-License-Identifier: Apache-2.0
+   */
 
 For example, `here are the instructions for adding copyright profiles in IntelliJ IDEA <https://www.jetbrains.com/help/idea/copyright.html>`__.
 
@@ -138,10 +132,10 @@ The plugin codebase is in standard layout of Gradle project::
    ├── build.gradle
    ├── config
    ├── docs
-   │   ├── attributions.md
-   │   ├── category.json
-   │   ├── dev
-   │   └── user
+   │   ├── attributions.md
+   │   ├── category.json
+   │   ├── dev
+   │   └── user
    ├── gradle.properties
    ├── gradlew
    ├── gradlew.bat
@@ -217,63 +211,21 @@ Java files are formatted using `Spotless <https://github.com/diffplug/spotless>`
    * - Javadoc format can be maintained by wrapping javadoc with `<pre></pre>` HTML tags
    * - Strings can be formatted on multiple lines with a `+` with the correct indentation for the string.
 
-New PPL Command Checklist
-=========================
+Development Guidelines
+----------------------
 
-If you are working on contributing a new PPL command, please read this guide and review all items in the checklist are done before code review. You also can leverage this checklist to guide how to add new PPL command.
+For detailed development documentation, please refer to the `development documentation <docs/dev/index.md>`_. For specific guidance on implementing PPL components, see the following resources:
 
-Prerequisite
-------------
-
-| ✅ Open an RFC issue before starting to code:
-- Describe the purpose of the new command
-- Include at least syntax definition, usage and examples
-- Implementation options are welcome if you have multiple ways to implement it
-| ✅ Obtain PM review approval for the RFC:
-- If PM unavailable, consult repository maintainers as alternative
-- An offline meeting might be required to discuss the syntax and usage
-
-Coding & Tests
---------------
-
-| ✅ Lexer/Parser Updates:
-- Add new keywords to OpenSearchPPLLexer.g4
-- Add grammar rules to OpenSearchPPLParser.g4
-- Update ``commandName`` and ``keywordsCanBeId``
-| ✅ AST Implementation:
-- Add new tree nodes under package ``org.opensearch.sql.ast.tree``
-- Prefer reusing ``Argument`` for command arguments **over** creating new expression nodes under ``org.opensearch.sql.ast.expression``
-| ✅ Visitor Pattern:
-- Add ``visit*`` in ``AbstractNodeVisitor``
-- Overriding ``visit*`` in ``Analyzer``, ``CalciteRelNodeVisitor`` and ``PPLQueryDataAnonymizer``
-| ✅ Unit Tests:
-- Extend ``CalcitePPLAbstractTest``
-- Keep test queries minimal
-- Include ``verifyLogical()`` and ``verifyPPLToSparkSQL()``
-| ✅ Integration tests (pushdown):
-- Extend ``PPLIntegTestCase``
-- Use complex real-world queries
-- Include ``verifySchema()`` and ``verifyDataRows()``
-| ✅ Integration tests (Non-pushdown):
-- Add test class to ``CalciteNoPushdownIT``
-| ✅ Explain tests:
-- Add tests to ``ExplainIT`` or ``CalciteExplainIT``
-| ✅ Unsupported in v2 test:
-- Add a test in ``NewAddedCommandsIT``
-| ✅ Anonymizer tests:
-- Add a test in ``PPLQueryDataAnonymizerTest``
-| ✅ Cross-cluster Tests (optional, nice to have):
-- Add a test in ``CrossClusterSearchIT``
-| ✅ User doc:
-- Add a xxx.rst under ``docs/user/ppl/cmd`` and link the new doc to ``docs/user/ppl/index.rst``
+- `PPL Commands <docs/dev/ppl-commands.md>`_: Guidelines for adding new commands to PPL
+- `PPL Functions <docs/dev/ppl-functions.md>`_: Instructions for implementing and integrating custom functions
 
 Building and Running Tests
 ==========================
 
 Gradle Build
 ------------
 
-Most of the time you just need to run ./gradlew build which will make sure you pass all checks and testing. While you’re developing, you may want to run specific Gradle task only. In this case, you can run ./gradlew with task name which only triggers the task along with those it depends on. Here is a list for common tasks:
+Most of the time you just need to run ./gradlew build which will make sure you pass all checks and testing. While you're developing, you may want to run specific Gradle task only. In this case, you can run ./gradlew with task name which only triggers the task along with those it depends on. Here is a list for common tasks:
 
 .. list-table::
    :widths: 30 50
@@ -294,7 +246,7 @@ Most of the time you just need to run ./gradlew build which will make sure you p
    * - ./gradlew :integ-test:yamlRestTest
      - Run rest integration test.
    * - ./gradlew :doctest:doctest
-     - Run doctests
+     - Run doctests in docs folder. You can use ``-Pdocs=file1,file2`` to run specific file(s). See more info in `Documentation <#documentation>`_ section.
    * - ./gradlew build
      - Build plugin by run all tasks above (this takes time).
    * - ./gradlew pitest
@@ -304,7 +256,7 @@ Most of the time you just need to run ./gradlew build which will make sure you p
    * - ./gradlew spotlessApply
      - Automatically apply spotless code style changes.
 
-For integration test, you can use ``-Dtests.class`` “UT full path” to run a task individually. For example ``./gradlew :integ-test:integTest -Dtests.class="*QueryIT"``.
+For integration test, you can use ``-Dtests.class`` "UT full path" to run a task individually. For example ``./gradlew :integ-test:integTest -Dtests.class="*QueryIT"``.
 
 To run the task above for specific module, you can do ``./gradlew :<module_name>:task``. For example, only build core module by ``./gradlew :core:build``.
 
@@ -466,6 +418,18 @@ Doctest
 
 Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
 
+.. code-block:: bash
+   # Test all docs
+   ./gradlew :doctest:doctest
+
+   # Test single file using main doctest task
+   ./gradlew :doctest:doctest -Pdocs=search
+   
+   # Test multiple files at once
+   ./gradlew :doctest:doctest -Pdocs=search,fields,basics
+   
+   # With verbose output
+   ./gradlew :doctest:doctest -Pdocs=stats -Pverbose=true
 
 Backports
 >>>>>>>>>

build.gradle

@@ -66,9 +66,9 @@ buildscript {
 
     repositories {
         mavenLocal()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
         mavenCentral()
+        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
     }
 
     dependencies {
@@ -92,10 +92,10 @@ apply plugin: 'opensearch.java-agent'
 // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{}
 repositories {
     mavenLocal()
-    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
+    maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
     maven { url 'https://jitpack.io' }
+    maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
 }
 
 spotless {
@@ -157,11 +157,10 @@ allprojects {
 subprojects {
     repositories {
         mavenLocal()
-        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
-        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
         mavenCentral()
-        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
+        maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
         maven { url 'https://jitpack.io' }
+        maven { url "https://ci.opensearch.org/ci/dbc/snapshots/" }
     }
 }
 

common/src/main/java/org/opensearch/sql/common/patterns/BrainLogParser.java

@@ -39,6 +39,11 @@ public class BrainLogParser {
             "(\\d{4}-\\d{2}-\\d{2})[T"
                 + " ]?(\\d{2}:\\d{2}:\\d{2})(\\.\\d{3})?(Z|([+-]\\d{2}:?\\d{2}))?"),
         "<*DATETIME*>");
+    // UUID
+    DEFAULT_FILTER_PATTERN_VARIABLE_MAP.put(
+        Pattern.compile(
+            "\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b"),
+        "<*UUID*>");
     // Hex Decimal, letters followed by digits, float numbers
     DEFAULT_FILTER_PATTERN_VARIABLE_MAP.put(
         Pattern.compile(

common/src/test/java/org/opensearch/sql/common/patterns/BrainLogParserTest.java

@@ -104,6 +104,22 @@ public void testPreprocess() {
     assertEquals(expectedResult, result);
   }
 
+  @Test
+  public void testPreprocessWithUUID() {
+    String logMessage = "127.0.0.1 - 1234 something, user_id:c78ac970-f0c3-4954-8cf8-352a8458d01c";
+    String logId = "log1";
+    List<String> expectedResult =
+        Arrays.asList("<*IP*>", "-", "<*>", "something", "user_id:<*UUID*>", "log1");
+    List<String> result = parser.preprocess(logMessage, logId);
+    assertEquals(expectedResult, result);
+    // Test with different delimiter
+    logMessage = "127.0.0.1=1234 something, user_id:c78ac970-f0c3-4954-8cf8-352a8458d01c";
+    logId = "log2";
+    expectedResult = Arrays.asList("<*IP*>=<*>", "something", "user_id:<*UUID*>", "log2");
+    result = parser.preprocess(logMessage, logId);
+    assertEquals(expectedResult, result);
+  }
+
   @Test
   public void testPreprocessWithIllegalInput() {
     String logMessage = "127.0.0.1 - 1234 something";

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -59,6 +59,7 @@
 import org.opensearch.sql.ast.tree.AD;
 import org.opensearch.sql.ast.tree.Aggregation;
 import org.opensearch.sql.ast.tree.AppendCol;
+import org.opensearch.sql.ast.tree.Bin;
 import org.opensearch.sql.ast.tree.CloseCursor;
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
@@ -669,6 +670,12 @@ public LogicalPlan visitML(ML node, AnalysisContext context) {
     return new LogicalML(child, node.getArguments());
   }
 
+  @Override
+  public LogicalPlan visitBin(Bin node, AnalysisContext context) {
+    throw new UnsupportedOperationException(
+        "Bin command is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
+  }
+
   @Override
   public LogicalPlan visitExpand(Expand expand, AnalysisContext context) {
     throw new UnsupportedOperationException(

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -47,6 +47,7 @@
 import org.opensearch.sql.ast.tree.AD;
 import org.opensearch.sql.ast.tree.Aggregation;
 import org.opensearch.sql.ast.tree.AppendCol;
+import org.opensearch.sql.ast.tree.Bin;
 import org.opensearch.sql.ast.tree.CloseCursor;
 import org.opensearch.sql.ast.tree.Dedupe;
 import org.opensearch.sql.ast.tree.Eval;
@@ -70,6 +71,7 @@
 import org.opensearch.sql.ast.tree.RelationSubquery;
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.Reverse;
+import org.opensearch.sql.ast.tree.SPath;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
 import org.opensearch.sql.ast.tree.TableFunction;
@@ -213,6 +215,10 @@ public T visitBetween(Between node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitBin(Bin node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitArgument(Argument node, C context) {
     return visitChildren(node, context);
   }
@@ -237,6 +243,10 @@ public T visitParse(Parse node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitSpath(SPath node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitLet(Let node, C context) {
     return visitChildren(node, context);
   }