Skip to content

Commit 4f1a0d7

Browse files
xinyualxinyual
committed
Merge remote-tracking branch 'origin/main' into addDistinctApprox
2 parents b90e7b8 + ed507d7 commit 4f1a0d7

43 files changed

Lines changed: 2089 additions & 265 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎async-query-core/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -868,7 +868,6 @@ jsonFunctionName
868868
| JSON_OBJECT
869869
| JSON_ARRAY
870870
| JSON_ARRAY_LENGTH
871-
| TO_JSON_STRING
872871
| JSON_EXTRACT
873872
| JSON_KEYS
874873
| JSON_VALID

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ repositories {
1414

1515
dependencies {
1616
implementation project(':core')
17+
implementation project(':opensearch')
1718

1819
// Dependencies required by JMH micro benchmark
1920
api group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.36'

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.operator.predicate;
7+
8+
import java.util.ArrayList;
9+
import java.util.HashMap;
10+
import java.util.LinkedHashMap;
11+
import java.util.List;
12+
import java.util.Map;
13+
import org.openjdk.jmh.annotations.Benchmark;
14+
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
15+
import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
16+
17+
public class MergeArrayAndObjectMapBenchmark {
18+
private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
19+
20+
@Benchmark
21+
public void testMerge() {
22+
Map<String, OpenSearchDataType> finalResult = new HashMap<>();
23+
for (Map<String, OpenSearchDataType> map : candidateMaps) {
24+
OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
25+
}
26+
}
27+
28+
private static Map<String, OpenSearchDataType> prepareMap(int recursive, String prefix) {
29+
Map<String, OpenSearchDataType> map = new HashMap<>();
30+
Map<String, Object> innerMap = prepareRecursiveMap(recursive, prefix);
31+
map.put("name", OpenSearchDataType.of(OpenSearchDataType.MappingType.Object, innerMap));
32+
return map;
33+
}
34+
35+
public static Map<String, Object> prepareRecursiveMap(int recursive, String prefix) {
36+
Map<String, Object> innerMap = new LinkedHashMap<>();
37+
if (recursive == 0) {
38+
innerMap.put("type", "string");
39+
} else {
40+
innerMap.put("type", "object");
41+
innerMap.put(
42+
"properties",
43+
Map.of(
44+
prefix + "_" + String.valueOf(recursive),
45+
Map.of("type", "text"),
46+
"recursive",
47+
prepareRecursiveMap(recursive - 1, prefix)));
48+
}
49+
return innerMap;
50+
}
51+
52+
private static List<Map<String, OpenSearchDataType>> prepareListOfMaps(int listNumber) {
53+
List<Map<String, OpenSearchDataType>> list = new ArrayList<>();
54+
for (int i = 0; i < listNumber; i++) {
55+
list.add(prepareMap(15, "prefix" + i));
56+
}
57+
return list;
58+
}
59+
}

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/PatternsWindowFunctionBenchmark.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.opensearch.sql.expression.ReferenceExpression;
3232
import org.opensearch.sql.expression.window.WindowDefinition;
3333
import org.opensearch.sql.expression.window.frame.BufferPatternRowsWindowFrame;
34-
import org.opensearch.sql.expression.window.frame.CurrentRowWindowFrame;
3534
import org.opensearch.sql.expression.window.frame.WindowFrame;
3635

3736
@Warmup(iterations = 1)
@@ -62,14 +61,6 @@ public class PatternsWindowFunctionBenchmark {
6261
new BrainLogParser(),
6362
new NamedArgumentExpression("message", new ReferenceExpression("message", STRING)));
6463

65-
@Benchmark
66-
public void testSimplePattern() {
67-
CurrentRowWindowFrame windowFrame =
68-
new CurrentRowWindowFrame(new WindowDefinition(ImmutableList.of(), ImmutableList.of()));
69-
70-
run(windowFrame, DSL.simple_pattern(DSL.ref("message", STRING)));
71-
}
72-
7364
@Benchmark
7465
public void testBrain() {
7566
BufferPatternRowsWindowFrame windowFrame =

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ allprojects {
129129
}
130130
configurations.all {
131131
resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10"
132+
resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}"
132133
resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10"
133134
resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.19"
134135
resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"

‎core/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ dependencies {
6363
api 'org.apache.calcite:calcite-linq4j:1.38.0'
6464
api project(':common')
6565
implementation "com.github.seancfoley:ipaddress:5.4.2"
66+
implementation "com.jayway.jsonpath:json-path:2.9.0"
67+
implementation "com.googlecode.aviator:aviator:5.4.3"
6668

6769
annotationProcessor('org.immutables:value:2.8.8')
6870
compileOnly('org.immutables:value-annotations:2.8.8')

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,13 @@ public enum BuiltinFunctionName {
170170
LIKE(FunctionName.of("like")),
171171
NOT_LIKE(FunctionName.of("not like")),
172172

173+
/** LAMBDA Functions * */
174+
ARRAY_FORALL(FunctionName.of("forall")),
175+
ARRAY_EXISTS(FunctionName.of("exists")),
176+
ARRAY_FILTER(FunctionName.of("filter")),
177+
ARRAY_TRANSFORM(FunctionName.of("transform")),
178+
ARRAY_AGGREGATE(FunctionName.of("reduce")),
179+
173180
/** Aggregation Function. */
174181
AVG(FunctionName.of("avg")),
175182
SUM(FunctionName.of("sum")),
@@ -213,9 +220,21 @@ public enum BuiltinFunctionName {
213220
TRIM(FunctionName.of("trim")),
214221
UPPER(FunctionName.of("upper")),
215222

223+
/** Array Functions. */
224+
ARRAY(FunctionName.of("array")),
225+
216226
/** Json Functions. */
217227
JSON_VALID(FunctionName.of("json_valid")),
218228
JSON(FunctionName.of("json")),
229+
JSON_OBJECT(FunctionName.of("json_object")),
230+
JSON_ARRAY(FunctionName.of("json_array")),
231+
JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")),
232+
JSON_EXTRACT(FunctionName.of("json_extract")),
233+
JSON_KEYS(FunctionName.of("json_keys")),
234+
JSON_SET(FunctionName.of("json_set")),
235+
JSON_DELETE(FunctionName.of("json_delete")),
236+
JSON_APPEND(FunctionName.of("json_append")),
237+
JSON_EXTEND(FunctionName.of("json_extend")),
219238

220239
/** GEOSPATIAL Functions. */
221240
GEOIP(FunctionName.of("geoip")),

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
2929
import org.opensearch.sql.expression.datetime.IntervalClause;
3030
import org.opensearch.sql.expression.ip.IPFunctions;
31-
import org.opensearch.sql.expression.json.JsonFunctions;
3231
import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions;
3332
import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions;
3433
import org.opensearch.sql.expression.operator.convert.TypeCastOperators;
@@ -84,7 +83,6 @@ public static synchronized BuiltinFunctionRepository getInstance() {
8483
SystemFunctions.register(instance);
8584
OpenSearchFunctions.register(instance);
8685
IPFunctions.register(instance);
87-
JsonFunctions.register(instance);
8886
}
8987
return instance;
9088
}

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@
3131
import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
3232
import org.opensearch.sql.data.type.ExprCoreType;
3333
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
34+
import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl;
35+
import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl;
36+
import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl;
37+
import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl;
38+
import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl;
39+
import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl;
40+
import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
41+
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
3442
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
3543
import org.opensearch.sql.expression.function.udf.GrokFunction;
3644
import org.opensearch.sql.expression.function.udf.SpanFunction;
@@ -64,6 +72,18 @@
6472
/** Defines functions and operators that are implemented only by PPL */
6573
public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
6674

75+
// Json Functions
76+
public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON");
77+
public static final SqlOperator JSON_ARRAY_LENGTH =
78+
new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH");
79+
public static final SqlOperator JSON_EXTRACT =
80+
new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT");
81+
public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS");
82+
public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET");
83+
public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE");
84+
public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND");
85+
public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND");
86+
6787
// Math functions
6888
public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN");
6989
public static final SqlOperator E = new EulerFunction().toUDF("E");

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.sql.expression.function;
77

8+
import static org.apache.calcite.sql.SqlJsonConstructorNullClause.NULL_ON_NULL;
89
import static org.apache.calcite.sql.type.SqlTypeFamily.IGNORE;
910
import static org.opensearch.sql.calcite.utils.CalciteToolsHelper.STDDEV_POP_NULLABLE;
1011
import static org.opensearch.sql.calcite.utils.CalciteToolsHelper.STDDEV_SAMP_NULLABLE;
@@ -80,6 +81,17 @@
8081
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL;
8182
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL;
8283
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_PRESENT;
84+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON;
85+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_APPEND;
86+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY;
87+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY_LENGTH;
88+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE;
89+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND;
90+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT;
91+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS;
92+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT;
93+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET;
94+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_VALID;
8395
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST_DAY;
8496
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT;
8597
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LENGTH;
@@ -194,6 +206,7 @@
194206
import java.util.concurrent.ConcurrentHashMap;
195207
import java.util.function.BiFunction;
196208
import java.util.stream.Collectors;
209+
import java.util.stream.Stream;
197210
import org.apache.calcite.rel.type.RelDataType;
198211
import org.apache.calcite.rex.RexBuilder;
199212
import org.apache.calcite.rex.RexNode;
@@ -735,6 +748,31 @@ void populate() {
735748
registerOperator(WEEK_OF_YEAR, PPLBuiltinOperators.WEEK);
736749
registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK);
737750

751+
// Register Json function
752+
register(
753+
JSON_ARRAY,
754+
((builder, args) ->
755+
builder.makeCall(
756+
SqlStdOperatorTable.JSON_ARRAY,
757+
Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args))
758+
.toArray(RexNode[]::new))));
759+
register(
760+
JSON_OBJECT,
761+
((builder, args) ->
762+
builder.makeCall(
763+
SqlStdOperatorTable.JSON_OBJECT,
764+
Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args))
765+
.toArray(RexNode[]::new))));
766+
registerOperator(JSON, PPLBuiltinOperators.JSON);
767+
registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH);
768+
registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT);
769+
registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS);
770+
registerOperator(JSON_VALID, SqlStdOperatorTable.IS_JSON_VALUE);
771+
registerOperator(JSON_SET, PPLBuiltinOperators.JSON_SET);
772+
registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE);
773+
registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND);
774+
registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND);
775+
738776
// Register implementation.
739777
// Note, make the implementation an individual class if too complex.
740778
register(

0 commit comments

Comments
 (0)