Merge remote-tracking branch 'origin/main' into issues/4004

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger @qianheng-aws
+*   @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @mengweieric @vamsimanohar @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger @qianheng-aws @yuancu

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,8 +10,9 @@ Resolves #[Issue number to be closed when this PR is merged]
 - [ ] New functionality has been documented.
  - [ ] New functionality has javadoc added.
  - [ ] New functionality has a user manual doc added.
+- [ ] New PPL command [checklist](https://github.com/opensearch-project/sql/blob/main/DEVELOPER_GUIDE.rst#new-ppl-command-checklist) all confirmed.
 - [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
-- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Commits are signed per the DCO using `--signoff` or `-s`.
 - [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

.github/maven-publish-utils.sh

@@ -4,6 +4,10 @@
 
 set -e
 
+# Flag to disable commit mapping functionality
+# Set to "true" to disable commit mapping operations
+DISABLE_COMMIT_MAPPING="${DISABLE_COMMIT_MAPPING:-true}"
+
 # Function to execute curl commands with retry and error handling
 execute_curl_with_retry() {
   local url="$1"
@@ -111,6 +115,11 @@ update_version_metadata() {
   local commit_id="$3"
   local snapshot_repo_url="${4:-$SNAPSHOT_REPO_URL}"
 
+  if [ "$DISABLE_COMMIT_MAPPING" = "true" ]; then
+    echo "Skipping version metadata update (commit mapping disabled)"
+    return 0
+  fi
+
   echo "Updating version metadata for ${artifact_id} version ${version} with commit ID ${commit_id}"
 
   TEMP_DIR=$(mktemp -d)
@@ -204,6 +213,11 @@ update_commit_mapping() {
   local commit_map_filename="${5:-$COMMIT_MAP_FILENAME}"
   local snapshot_repo_url="${6:-$SNAPSHOT_REPO_URL}"
 
+  if [ "$DISABLE_COMMIT_MAPPING" = "true" ]; then
+    echo "Skipping commit-version mapping update (commit mapping disabled)"
+    return 0
+  fi
+
   echo "Updating commit-version mapping for ${artifact_id}"
 
   # Create temp directory for work

DEVELOPER_GUIDE.rst

@@ -217,6 +217,56 @@ Java files are formatted using `Spotless <https://github.com/diffplug/spotless>`
    * - Javadoc format can be maintained by wrapping javadoc with `<pre></pre>` HTML tags
    * - Strings can be formatted on multiple lines with a `+` with the correct indentation for the string.
 
+New PPL Command Checklist
+=========================
+
+If you are working on contributing a new PPL command, please read this guide and review all items in the checklist are done before code review. You also can leverage this checklist to guide how to add new PPL command.
+
+Prerequisite
+------------
+
+| ✅ Open an RFC issue before starting to code:
+- Describe the purpose of the new command
+- Include at least syntax definition, usage and examples
+- Implementation options are welcome if you have multiple ways to implement it
+| ✅ Obtain PM review approval for the RFC:
+- If PM unavailable, consult repository maintainers as alternative
+- An offline meeting might be required to discuss the syntax and usage
+
+Coding & Tests
+--------------
+
+| ✅ Lexer/Parser Updates:
+- Add new keywords to OpenSearchPPLLexer.g4
+- Add grammar rules to OpenSearchPPLParser.g4
+- Update ``commandName`` and ``keywordsCanBeId``
+| ✅ AST Implementation:
+- Add new tree nodes under package ``org.opensearch.sql.ast.tree``
+- Prefer reusing ``Argument`` for command arguments **over** creating new expression nodes under ``org.opensearch.sql.ast.expression``
+| ✅ Visitor Pattern:
+- Add ``visit*`` in ``AbstractNodeVisitor``
+- Overriding ``visit*`` in ``Analyzer``, ``CalciteRelNodeVisitor`` and ``PPLQueryDataAnonymizer``
+| ✅ Unit Tests:
+- Extend ``CalcitePPLAbstractTest``
+- Keep test queries minimal
+- Include ``verifyLogical()`` and ``verifyPPLToSparkSQL()``
+| ✅ Integration tests (pushdown):
+- Extend ``PPLIntegTestCase``
+- Use complex real-world queries
+- Include ``verifySchema()`` and ``verifyDataRows()``
+| ✅ Integration tests (Non-pushdown):
+- Add test class to ``CalciteNoPushdownIT``
+| ✅ Explain tests:
+- Add tests to ``ExplainIT`` or ``CalciteExplainIT``
+| ✅ Unsupported in v2 test:
+- Add a test in ``NewAddedCommandsIT``
+| ✅ Anonymizer tests:
+- Add a test in ``PPLQueryDataAnonymizerTest``
+| ✅ Cross-cluster Tests (optional, nice to have):
+- Add a test in ``CrossClusterSearchIT``
+| ✅ User doc:
+- Add a xxx.rst under ``docs/user/ppl/cmd`` and link the new doc to ``docs/user/ppl/index.rst``
+
 Building and Running Tests
 ==========================
 

MAINTAINERS.md

@@ -10,10 +10,8 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Joshua Li         | [joshuali925](https://github.com/joshuali925)       | Amazon      |
 | Shenoy Pratik     | [ps48](https://github.com/ps48)                     | Amazon      |
 | Kavitha Mohan     | [kavithacm](https://github.com/kavithacm)           | Amazon      |
-| Rupal Mahajan     | [rupal-bq](https://github.com/rupal-bq)             | Amazon      |
 | Derek Ho          | [derek-ho](https://github.com/derek-ho)             | Amazon      |
 | Lior Perry        | [YANG-DB](https://github.com/YANG-DB)               | Amazon      |
-| Peter Fitzgibbons | [pjfitzgibbons](https://github.com/pjfitzgibbons)   | Amazon      |
 | Simeon Widdis     | [swiddis](https://github.com/swiddis)               | Amazon      |
 | Chen Dai          | [dai-chen](https://github.com/dai-chen)             | Amazon      |
 | Vamsi Manohar     | [vamsimanohar](https://github.com/vamsimanohar)     | Amazon      |
@@ -24,6 +22,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Lantao Jin        | [LantaoJin](https://github.com/LantaoJin)           | Amazon      |
 | Louis Chu         | [noCharger](https://github.com/noCharger)           | Amazon      |
 | Heng Qian         | [qianheng-aws](https://github.com/qianheng-aws)     | Amazon      |
+| Yuanchun Shen     | [yuancu](https://github.com/yuancu)                 | Amazon      |
 | Max Ksyunz        | [MaxKsyunz](https://github.com/MaxKsyunz)           | Improving   |
 | Yury Fridlyand    | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | Improving   |
 | Andrew Carbonetto | [acarbonetto](https://github.com/acarbonetto)       | Improving   |
@@ -40,3 +39,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Eugene Lee        | [eugenesk24](https://github.com/eugenesk24)             | Amazon      |
 | Zhongnan Su       | [zhongnansu](https://github.com/zhongnansu)             | Amazon      |
 | Chloe Zhang       | [chloe-zh](https://github.com/chloe-zh)                 | Amazon      |
+| Peter Fitzgibbons | [pjfitzgibbons](https://github.com/pjfitzgibbons)       | Amazon      |
+| Rupal Mahajan     | [rupal-bq](https://github.com/rupal-bq)                 | Amazon      |
+

async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java

@@ -43,14 +43,24 @@ public class SQLQueryUtils {
   private static final Logger logger = LogManager.getLogger(SQLQueryUtils.class);
 
   public static List<FullyQualifiedTableName> extractFullyQualifiedTableNames(String sqlQuery) {
-    SqlBaseParser sqlBaseParser =
-        new SqlBaseParser(
-            new CommonTokenStream(new SqlBaseLexer(new CaseInsensitiveCharStream(sqlQuery))));
-    sqlBaseParser.addErrorListener(new SyntaxAnalysisErrorListener());
+    return extractFullyQualifiedTableNamesWithMetadata(sqlQuery).getFullyQualifiedTableNames();
+  }
+
+  public static TableExtractionResult extractFullyQualifiedTableNamesWithMetadata(String sqlQuery) {
+    SqlBaseParser sqlBaseParser = getBaseParser(sqlQuery);
     StatementContext statement = sqlBaseParser.statement();
-    SparkSqlTableNameVisitor sparkSqlTableNameVisitor = new SparkSqlTableNameVisitor();
-    statement.accept(sparkSqlTableNameVisitor);
-    return sparkSqlTableNameVisitor.getFullyQualifiedTableNames();
+    SparkSqlTableNameVisitor visitor = new SparkSqlTableNameVisitor();
+    statement.accept(visitor);
+
+    // Remove duplicate table names
+    List<FullyQualifiedTableName> uniqueFullyQualifiedTableNames = new LinkedList<>();
+    for (FullyQualifiedTableName fullyQualifiedTableName : visitor.getFullyQualifiedTableNames()) {
+      if (!uniqueFullyQualifiedTableNames.contains(fullyQualifiedTableName)) {
+        uniqueFullyQualifiedTableNames.add(fullyQualifiedTableName);
+      }
+    }
+
+    return new TableExtractionResult(uniqueFullyQualifiedTableNames, visitor.isCreateTable());
   }
 
   public static IndexQueryDetails extractIndexDetails(String sqlQuery) {
@@ -93,6 +103,8 @@ public static class SparkSqlTableNameVisitor extends SqlBaseParserBaseVisitor<Vo
     @Getter
     private final List<FullyQualifiedTableName> fullyQualifiedTableNames = new LinkedList<>();
 
+    @Getter private boolean isCreateTable = false;
+
     public SparkSqlTableNameVisitor() {}
 
     @Override
@@ -131,6 +143,12 @@ public Void visitCreateTableHeader(SqlBaseParser.CreateTableHeaderContext ctx) {
       }
       return super.visitCreateTableHeader(ctx);
     }
+
+    @Override
+    public Void visitCreateTable(SqlBaseParser.CreateTableContext ctx) {
+      isCreateTable = true;
+      return super.visitCreateTable(ctx);
+    }
   }
 
   public static class FlintSQLIndexDetailsVisitor extends FlintSparkSqlExtensionsBaseVisitor<Void> {
@@ -381,4 +399,15 @@ public String removeUnwantedQuotes(String input) {
       return input.replaceAll("^\"|\"$", "");
     }
   }
+
+  public static class TableExtractionResult {
+    @Getter private final List<FullyQualifiedTableName> fullyQualifiedTableNames;
+    @Getter private final boolean isCreateTableQuery;
+
+    public TableExtractionResult(
+        List<FullyQualifiedTableName> fullyQualifiedTableNames, boolean isCreateTableQuery) {
+      this.fullyQualifiedTableNames = fullyQualifiedTableNames;
+      this.isCreateTableQuery = isCreateTableQuery;
+    }
+  }
 }

async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java

@@ -25,6 +25,7 @@
 import org.opensearch.sql.spark.dispatcher.model.IndexQueryActionType;
 import org.opensearch.sql.spark.dispatcher.model.IndexQueryDetails;
 import org.opensearch.sql.spark.flint.FlintIndexType;
+import org.opensearch.sql.spark.utils.SQLQueryUtils.TableExtractionResult;
 
 @ExtendWith(MockitoExtension.class)
 public class SQLQueryUtilsTest {
@@ -444,6 +445,69 @@ void testRecoverIndex() {
     assertEquals(IndexQueryActionType.RECOVER, indexDetails.getIndexQueryActionType());
   }
 
+  @Test
+  void testExtractFullyQualifiedTableNamesWithMetadata() {
+    // Test CREATE TABLE queries
+    String createTableQuery =
+        "CREATE EXTERNAL TABLE\n"
+            + "myS3.default.alb_logs\n"
+            + "[ PARTITIONED BY (col_name [, … ] ) ]\n"
+            + "[ ROW FORMAT DELIMITED row_format ]\n"
+            + "STORED AS file_format\n"
+            + "LOCATION { 's3://bucket/folder/' }";
+
+    TableExtractionResult result =
+        SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(createTableQuery);
+    assertTrue(result.isCreateTableQuery());
+    assertEquals(1, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "alb_logs", result.getFullyQualifiedTableNames().get(0));
+
+    String createTableQuery2 =
+        "CREATE TABLE myS3.default.new_table (id INT, name STRING) USING PARQUET";
+    result = SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(createTableQuery2);
+    assertTrue(result.isCreateTableQuery());
+    assertEquals(1, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "new_table", result.getFullyQualifiedTableNames().get(0));
+
+    // Test SELECT queries
+    String selectQuery = "SELECT * FROM myS3.default.alb_logs";
+    result = SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(selectQuery);
+    assertFalse(result.isCreateTableQuery());
+    assertEquals(1, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "alb_logs", result.getFullyQualifiedTableNames().get(0));
+
+    // Test DROP TABLE queries
+    String dropTableQuery = "DROP TABLE myS3.default.alb_logs";
+    result = SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(dropTableQuery);
+    assertFalse(result.isCreateTableQuery());
+    assertEquals(1, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "alb_logs", result.getFullyQualifiedTableNames().get(0));
+
+    // Test DESCRIBE TABLE queries
+    String describeTableQuery = "DESCRIBE TABLE myS3.default.alb_logs";
+    result = SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(describeTableQuery);
+    assertFalse(result.isCreateTableQuery());
+    assertEquals(1, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "alb_logs", result.getFullyQualifiedTableNames().get(0));
+
+    // Test JOIN queries
+    String joinQuery =
+        "SELECT * FROM myS3.default.alb_logs JOIN myS3.default.http_logs ON alb_logs.id ="
+            + " http_logs.id";
+    result = SQLQueryUtils.extractFullyQualifiedTableNamesWithMetadata(joinQuery);
+    assertFalse(result.isCreateTableQuery());
+    assertEquals(2, result.getFullyQualifiedTableNames().size());
+    assertFullyQualifiedTableName(
+        "myS3", "default", "alb_logs", result.getFullyQualifiedTableNames().get(0));
+    assertFullyQualifiedTableName(
+        "myS3", "default", "http_logs", result.getFullyQualifiedTableNames().get(1));
+  }
+
   @Getter
   protected static class IndexQuery {
     private String query;

common/src/main/java/org/opensearch/sql/common/patterns/BrainLogParser.java

@@ -39,6 +39,11 @@ public class BrainLogParser {
             "(\\d{4}-\\d{2}-\\d{2})[T"
                 + " ]?(\\d{2}:\\d{2}:\\d{2})(\\.\\d{3})?(Z|([+-]\\d{2}:?\\d{2}))?"),
         "<*DATETIME*>");
+    // UUID
+    DEFAULT_FILTER_PATTERN_VARIABLE_MAP.put(
+        Pattern.compile(
+            "\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b"),
+        "<*UUID*>");
     // Hex Decimal, letters followed by digits, float numbers
     DEFAULT_FILTER_PATTERN_VARIABLE_MAP.put(
         Pattern.compile(

common/src/test/java/org/opensearch/sql/common/patterns/BrainLogParserTest.java

@@ -104,6 +104,22 @@ public void testPreprocess() {
     assertEquals(expectedResult, result);
   }
 
+  @Test
+  public void testPreprocessWithUUID() {
+    String logMessage = "127.0.0.1 - 1234 something, user_id:c78ac970-f0c3-4954-8cf8-352a8458d01c";
+    String logId = "log1";
+    List<String> expectedResult =
+        Arrays.asList("<*IP*>", "-", "<*>", "something", "user_id:<*UUID*>", "log1");
+    List<String> result = parser.preprocess(logMessage, logId);
+    assertEquals(expectedResult, result);
+    // Test with different delimiter
+    logMessage = "127.0.0.1=1234 something, user_id:c78ac970-f0c3-4954-8cf8-352a8458d01c";
+    logId = "log2";
+    expectedResult = Arrays.asList("<*IP*>=<*>", "something", "user_id:<*UUID*>", "log2");
+    result = parser.preprocess(logMessage, logId);
+    assertEquals(expectedResult, result);
+  }
+
   @Test
   public void testPreprocessWithIllegalInput() {
     String logMessage = "127.0.0.1 - 1234 something";