Merge remote-tracking branch 'refs/remotes/origin/main' into poc/graphlookup

# Conflicts:
#	core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
#	core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java
#	docs/user/ppl/index.md
#	ppl/src/main/antlr/OpenSearchPPLParser.g4

Author: qianheng-aws

async-query-core/build.gradle

@@ -43,7 +43,7 @@ configurations {
 }
 
 dependencies {
-    antlr "org.antlr:antlr4:4.7.1"
+    antlr "org.antlr:antlr4:4.13.2"
 
     implementation project(':core')
     implementation 'org.json:json:20231013'

build.gradle

@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.5.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.6.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')

common/build.gradle

@@ -33,7 +33,7 @@ repositories {
 }
 
 dependencies {
-    api "org.antlr:antlr4-runtime:4.7.1"
+    api "org.antlr:antlr4-runtime:4.13.2"
     api group: 'com.google.guava', name: 'guava', version: "${guava_version}"
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:"${versions.log4j}"
     api group: 'org.apache.commons', name: 'commons-lang3', version: "${commons_lang3_version}"

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -82,6 +82,8 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Multisearch;
 import org.opensearch.sql.ast.tree.MvCombine;
+import org.opensearch.sql.ast.tree.MvExpand;
+import org.opensearch.sql.ast.tree.NoMv;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
@@ -547,6 +549,16 @@ public LogicalPlan visitMvCombine(MvCombine node, AnalysisContext context) {
     throw getOnlyForCalciteException("mvcombine");
   }
 
+  @Override
+  public LogicalPlan visitNoMv(NoMv node, AnalysisContext context) {
+    throw getOnlyForCalciteException("nomv");
+  }
+
+  @Override
+  public LogicalPlan visitMvExpand(MvExpand node, AnalysisContext context) {
+    throw getOnlyForCalciteException("mvexpand");
+  }
+
   @Override
   public LogicalPlan visitGraphLookup(GraphLookup node, AnalysisContext context) {
     throw getOnlyForCalciteException("graphlookup");

core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java

@@ -70,6 +70,8 @@
 import org.opensearch.sql.ast.tree.ML;
 import org.opensearch.sql.ast.tree.Multisearch;
 import org.opensearch.sql.ast.tree.MvCombine;
+import org.opensearch.sql.ast.tree.MvExpand;
+import org.opensearch.sql.ast.tree.NoMv;
 import org.opensearch.sql.ast.tree.Paginate;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
@@ -477,6 +479,14 @@ public T visitMvCombine(MvCombine node, C context) {
     return visitChildren(node, context);
   }
 
+  public T visitNoMv(NoMv node, C context) {
+    return visitChildren(node, context);
+  }
+
+  public T visitMvExpand(MvExpand node, C context) {
+    return visitChildren(node, context);
+  }
+
   public T visitGraphLookup(GraphLookup node, C context) {
     return visitChildren(node, context);
   }

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -63,6 +63,7 @@
 import org.opensearch.sql.ast.tree.Limit;
 import org.opensearch.sql.ast.tree.MinSpanBin;
 import org.opensearch.sql.ast.tree.MvCombine;
+import org.opensearch.sql.ast.tree.MvExpand;
 import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Patterns;
 import org.opensearch.sql.ast.tree.Project;
@@ -477,6 +478,16 @@ public static MvCombine mvcombine(Field field, String delim) {
     return new MvCombine(field, delim);
   }
 
+  /**
+   * Build an MVEXPAND plan node and attach it to the input plan.
+   *
+   * <p>`@param` input input plan `@param` field field to expand `@param` limit optional
+   * per-document limit `@return` MvExpand plan attached to the input
+   */
+  public static UnresolvedPlan mvexpand(UnresolvedPlan input, Field field, Integer limit) {
+    return new MvExpand(field, limit).attach(input);
+  }
+
   public static List<Argument> sortOptions() {
     return exprList(argument("desc", booleanLiteral(false)));
   }

core/src/main/java/org/opensearch/sql/ast/tree/MvExpand.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import javax.annotation.Nullable;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.Field;
+
+/** AST node representing the {@code mvexpand} PPL command: {@code mvexpand <field> [limit=N]}. */
+@ToString
+@EqualsAndHashCode(callSuper = false)
+public class MvExpand extends UnresolvedPlan {
+
+  private UnresolvedPlan child;
+  @Getter private final Field field;
+  @Getter @Nullable private final Integer limit;
+
+  public MvExpand(Field field, @Nullable Integer limit) {
+    this.field = field;
+    this.limit = limit;
+  }
+
+  @Override
+  public MvExpand attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitMvExpand(this, context);
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/NoMv.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ast.tree;
+
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import javax.annotation.Nullable;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+import org.opensearch.sql.ast.AbstractNodeVisitor;
+import org.opensearch.sql.ast.expression.DataType;
+import org.opensearch.sql.ast.expression.Field;
+import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.Let;
+import org.opensearch.sql.ast.expression.Literal;
+
+/**
+ * AST node for the NOMV command. Converts multi-value fields to single-value fields by joining
+ * array elements with newline delimiter.
+ */
+@Getter
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = false)
+public class NoMv extends UnresolvedPlan {
+
+  private final Field field;
+  @Nullable private UnresolvedPlan child;
+
+  public NoMv(Field field) {
+    this.field = field;
+  }
+
+  public NoMv attach(UnresolvedPlan child) {
+    this.child = child;
+    return this;
+  }
+
+  @Override
+  public List<UnresolvedPlan> getChild() {
+    return child == null ? ImmutableList.of() : ImmutableList.of(child);
+  }
+
+  @Override
+  public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
+    return nodeVisitor.visitNoMv(this, context);
+  }
+
+  /**
+   * Rewrites the nomv command as an eval command using mvjoin function with null filtering. nomv
+   * <field> is rewritten to: eval <field> = coalesce(mvjoin(array_compact(<field>), "\n"), "")
+   *
+   * <p>The array_compact removes null elements from the array, and coalesce ensures empty arrays
+   * return empty string instead of null.
+   *
+   * @return an Eval node representing the equivalent mvjoin operation with null filtering
+   */
+  public UnresolvedPlan rewriteAsEval() {
+    Function arrayCompactFunc = new Function("array_compact", ImmutableList.of(field));
+
+    Function mvjoinFunc =
+        new Function(
+            "mvjoin", ImmutableList.of(arrayCompactFunc, new Literal("\n", DataType.STRING)));
+
+    Function coalesceFunc =
+        new Function("coalesce", ImmutableList.of(mvjoinFunc, new Literal("", DataType.STRING)));
+
+    Let letExpr = new Let(field, coalesceFunc);
+
+    Eval eval = new Eval(ImmutableList.of(letExpr));
+    if (this.child != null) {
+      eval.attach(this.child);
+    }
+    return eval;
+  }
+}

core/src/main/java/org/opensearch/sql/ast/tree/SPath.java

@@ -30,7 +30,7 @@ public class SPath extends UnresolvedPlan {
 
   @Nullable private final String outField;
 
-  private final String path;
+  @Nullable private final String path;
 
   @Override
   public UnresolvedPlan attach(UnresolvedPlan child) {
@@ -48,7 +48,20 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
     return nodeVisitor.visitSpath(this, context);
   }
 
+  /**
+   * Rewrites this spath node to an equivalent {@link Eval} node.
+   *
+   * <p>In path mode, rewrites to {@code eval output = json_extract(input, path)}. In auto-extract
+   * mode (path is null), rewrites to {@code eval output = json_extract_all(input)}.
+   */
   public Eval rewriteAsEval() {
+    if (path != null) {
+      return rewritePathMode();
+    }
+    return rewriteAutoExtractMode();
+  }
+
+  private Eval rewritePathMode() {
     String outField = this.outField;
     String unquotedPath = unquoteText(this.path);
     if (outField == null) {
@@ -62,4 +75,12 @@ public Eval rewriteAsEval() {
             AstDSL.function(
                 "json_extract", AstDSL.field(inField), AstDSL.stringLiteral(unquotedPath))));
   }
+
+  private Eval rewriteAutoExtractMode() {
+    String output = (outField != null) ? outField : inField;
+    return AstDSL.eval(
+        child,
+        AstDSL.let(
+            AstDSL.field(output), AstDSL.function("json_extract_all", AstDSL.field(inField))));
+  }
 }