Merge origin/main into cmd-expand

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

async-query-core/src/main/antlr/OpenSearchPPLParser.g4

@@ -868,7 +868,6 @@ jsonFunctionName
    | JSON_OBJECT
    | JSON_ARRAY
    | JSON_ARRAY_LENGTH
-   | TO_JSON_STRING
    | JSON_EXTRACT
    | JSON_KEYS
    | JSON_VALID

benchmarks/build.gradle

@@ -14,6 +14,7 @@ repositories {
 
 dependencies {
     implementation project(':core')
+    implementation project(':opensearch')
 
     // Dependencies required by JMH micro benchmark
     api group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.36'

benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.expression.operator.predicate;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
+import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
+
+public class MergeArrayAndObjectMapBenchmark {
+  private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
+
+  @Benchmark
+  public void testMerge() {
+    Map<String, OpenSearchDataType> finalResult = new HashMap<>();
+    for (Map<String, OpenSearchDataType> map : candidateMaps) {
+      OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
+    }
+  }
+
+  private static Map<String, OpenSearchDataType> prepareMap(int recursive, String prefix) {
+    Map<String, OpenSearchDataType> map = new HashMap<>();
+    Map<String, Object> innerMap = prepareRecursiveMap(recursive, prefix);
+    map.put("name", OpenSearchDataType.of(OpenSearchDataType.MappingType.Object, innerMap));
+    return map;
+  }
+
+  public static Map<String, Object> prepareRecursiveMap(int recursive, String prefix) {
+    Map<String, Object> innerMap = new LinkedHashMap<>();
+    if (recursive == 0) {
+      innerMap.put("type", "string");
+    } else {
+      innerMap.put("type", "object");
+      innerMap.put(
+          "properties",
+          Map.of(
+              prefix + "_" + String.valueOf(recursive),
+              Map.of("type", "text"),
+              "recursive",
+              prepareRecursiveMap(recursive - 1, prefix)));
+    }
+    return innerMap;
+  }
+
+  private static List<Map<String, OpenSearchDataType>> prepareListOfMaps(int listNumber) {
+    List<Map<String, OpenSearchDataType>> list = new ArrayList<>();
+    for (int i = 0; i < listNumber; i++) {
+      list.add(prepareMap(15, "prefix" + i));
+    }
+    return list;
+  }
+}

benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/PatternsWindowFunctionBenchmark.java

@@ -31,7 +31,6 @@
 import org.opensearch.sql.expression.ReferenceExpression;
 import org.opensearch.sql.expression.window.WindowDefinition;
 import org.opensearch.sql.expression.window.frame.BufferPatternRowsWindowFrame;
-import org.opensearch.sql.expression.window.frame.CurrentRowWindowFrame;
 import org.opensearch.sql.expression.window.frame.WindowFrame;
 
 @Warmup(iterations = 1)
@@ -62,14 +61,6 @@ public class PatternsWindowFunctionBenchmark {
           new BrainLogParser(),
           new NamedArgumentExpression("message", new ReferenceExpression("message", STRING)));
 
-  @Benchmark
-  public void testSimplePattern() {
-    CurrentRowWindowFrame windowFrame =
-        new CurrentRowWindowFrame(new WindowDefinition(ImmutableList.of(), ImmutableList.of()));
-
-    run(windowFrame, DSL.simple_pattern(DSL.ref("message", STRING)));
-  }
-
   @Benchmark
   public void testBrain() {
     BufferPatternRowsWindowFrame windowFrame =

build.gradle

@@ -129,6 +129,7 @@ allprojects {
     }
     configurations.all {
         resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10"
+        resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}"
         resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10"
         resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.19"
         resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"

core/build.gradle

@@ -63,6 +63,8 @@ dependencies {
     api 'org.apache.calcite:calcite-linq4j:1.38.0'
     api project(':common')
     implementation "com.github.seancfoley:ipaddress:5.4.2"
+    implementation "com.jayway.jsonpath:json-path:2.9.0"
+    implementation "com.googlecode.aviator:aviator:5.4.3"
 
     annotationProcessor('org.immutables:value:2.8.8')
     compileOnly('org.immutables:value-annotations:2.8.8')

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -170,6 +170,13 @@ public enum BuiltinFunctionName {
   LIKE(FunctionName.of("like")),
   NOT_LIKE(FunctionName.of("not like")),
 
+  /** LAMBDA Functions * */
+  ARRAY_FORALL(FunctionName.of("forall")),
+  ARRAY_EXISTS(FunctionName.of("exists")),
+  ARRAY_FILTER(FunctionName.of("filter")),
+  ARRAY_TRANSFORM(FunctionName.of("transform")),
+  ARRAY_AGGREGATE(FunctionName.of("reduce")),
+
   /** Aggregation Function. */
   AVG(FunctionName.of("avg")),
   SUM(FunctionName.of("sum")),
@@ -212,9 +219,21 @@ public enum BuiltinFunctionName {
   TRIM(FunctionName.of("trim")),
   UPPER(FunctionName.of("upper")),
 
+  /** Array Functions. */
+  ARRAY(FunctionName.of("array")),
+
   /** Json Functions. */
   JSON_VALID(FunctionName.of("json_valid")),
   JSON(FunctionName.of("json")),
+  JSON_OBJECT(FunctionName.of("json_object")),
+  JSON_ARRAY(FunctionName.of("json_array")),
+  JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")),
+  JSON_EXTRACT(FunctionName.of("json_extract")),
+  JSON_KEYS(FunctionName.of("json_keys")),
+  JSON_SET(FunctionName.of("json_set")),
+  JSON_DELETE(FunctionName.of("json_delete")),
+  JSON_APPEND(FunctionName.of("json_append")),
+  JSON_EXTEND(FunctionName.of("json_extend")),
 
   /** GEOSPATIAL Functions. */
   GEOIP(FunctionName.of("geoip")),

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java

@@ -28,7 +28,6 @@
 import org.opensearch.sql.expression.datetime.DateTimeFunctions;
 import org.opensearch.sql.expression.datetime.IntervalClause;
 import org.opensearch.sql.expression.ip.IPFunctions;
-import org.opensearch.sql.expression.json.JsonFunctions;
 import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions;
 import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions;
 import org.opensearch.sql.expression.operator.convert.TypeCastOperators;
@@ -84,7 +83,6 @@ public static synchronized BuiltinFunctionRepository getInstance() {
       SystemFunctions.register(instance);
       OpenSearchFunctions.register(instance);
       IPFunctions.register(instance);
-      JsonFunctions.register(instance);
     }
     return instance;
   }

core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java

@@ -31,6 +31,14 @@
 import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.expression.datetime.DateTimeFunctions;
+import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
+import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
 import org.opensearch.sql.expression.function.udf.CryptographicFunction;
 import org.opensearch.sql.expression.function.udf.GrokFunction;
 import org.opensearch.sql.expression.function.udf.SpanFunction;
@@ -64,6 +72,18 @@
 /** Defines functions and operators that are implemented only by PPL */
 public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
 
+  // Json Functions
+  public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON");
+  public static final SqlOperator JSON_ARRAY_LENGTH =
+      new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH");
+  public static final SqlOperator JSON_EXTRACT =
+      new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT");
+  public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS");
+  public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET");
+  public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE");
+  public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND");
+  public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND");
+
   // Math functions
   public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN");
   public static final SqlOperator E = new EulerFunction().toUDF("E");