Skip to content

Commit 05aa651

Browse files
xinyualxinyual
authored
add json functions (opensearch-project#3559)
* add json functions Signed-off-by: xinyual <xinyual@amazon.com> * add new functions Signed-off-by: xinyual <xinyual@amazon.com> * add json related function Signed-off-by: xinyual <xinyual@amazon.com> * add all functions Signed-off-by: xinyual <xinyual@amazon.com> * add json functions Signed-off-by: xinyual <xinyual@amazon.com> * fix IT Signed-off-by: xinyual <xinyual@amazon.com> * fix error message Signed-off-by: xinyual <xinyual@amazon.com> * add json Signed-off-by: xinyual <xinyual@amazon.com> * try field Signed-off-by: xinyual <xinyual@amazon.com> * add json Signed-off-by: xinyual <xinyual@amazon.com> * update to splunk for extract Signed-off-by: xinyual <xinyual@amazon.com> * align with splunk Signed-off-by: xinyual <xinyual@amazon.com> * fi original IT Signed-off-by: xinyual <xinyual@amazon.com> * change array and json function Signed-off-by: xinyual <xinyual@amazon.com> * return string Signed-off-by: xinyual <xinyual@amazon.com> * apply spotless Signed-off-by: xinyual <xinyual@amazon.com> * revert useless change Signed-off-by: xinyual <xinyual@amazon.com> * apply spotless Signed-off-by: xinyual <xinyual@amazon.com> * revert useless change Signed-off-by: xinyual <xinyual@amazon.com> * revert useless change Signed-off-by: xinyual <xinyual@amazon.com> * align with splunk Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * change extract implementation Signed-off-by: xinyual <xinyual@amazon.com> * apply spotles Signed-off-by: xinyual <xinyual@amazon.com> * add more IT Signed-off-by: xinyual <xinyual@amazon.com> * remove to json string Signed-off-by: xinyual <xinyual@amazon.com> * remove useless change Signed-off-by: xinyual <xinyual@amazon.com> * add UT Signed-off-by: xinyual <xinyual@amazon.com> * add some doc Signed-off-by: xinyual <xinyual@amazon.com> * add docs Signed-off-by: xinyual <xinyual@amazon.com> * fix doc Signed-off-by: xinyual <xinyual@amazon.com> * add json path Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * optimize doc Signed-off-by: xinyual <xinyual@amazon.com> * add version and limitation to docs Signed-off-by: xinyual <xinyual@amazon.com> * remove original implementation of V2 Signed-off-by: xinyual <xinyual@amazon.com> * revert useless change Signed-off-by: xinyual <xinyual@amazon.com> * remove useless UTs Signed-off-by: xinyual <xinyual@amazon.com> * Ignore useless IT Signed-off-by: xinyual <xinyual@amazon.com> * update doc Signed-off-by: xinyual <xinyual@amazon.com> * add type checker for json Signed-off-by: xinyual <xinyual@amazon.com> --------- Signed-off-by: xinyual <xinyual@amazon.com>
1 parent 270aa0d commit 05aa651

24 files changed

Lines changed: 1797 additions & 240 deletions

‎async-query-core/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -868,7 +868,6 @@ jsonFunctionName
868868
| JSON_OBJECT
869869
| JSON_ARRAY
870870
| JSON_ARRAY_LENGTH
871-
| TO_JSON_STRING
872871
| JSON_EXTRACT
873872
| JSON_KEYS
874873
| JSON_VALID

‎build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ allprojects {
129129
}
130130
configurations.all {
131131
resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10"
132+
resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}"
132133
resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10"
133134
resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.19"
134135
resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"

‎core/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ dependencies {
6363
api 'org.apache.calcite:calcite-linq4j:1.38.0'
6464
api project(':common')
6565
implementation "com.github.seancfoley:ipaddress:5.4.2"
66+
implementation "com.jayway.jsonpath:json-path:2.9.0"
67+
implementation "com.googlecode.aviator:aviator:5.4.3"
6668

6769
annotationProcessor('org.immutables:value:2.8.8')
6870
compileOnly('org.immutables:value-annotations:2.8.8')

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,13 @@ public enum BuiltinFunctionName {
170170
LIKE(FunctionName.of("like")),
171171
NOT_LIKE(FunctionName.of("not like")),
172172

173+
/** LAMBDA Functions * */
174+
ARRAY_FORALL(FunctionName.of("forall")),
175+
ARRAY_EXISTS(FunctionName.of("exists")),
176+
ARRAY_FILTER(FunctionName.of("filter")),
177+
ARRAY_TRANSFORM(FunctionName.of("transform")),
178+
ARRAY_AGGREGATE(FunctionName.of("reduce")),
179+
173180
/** Aggregation Function. */
174181
AVG(FunctionName.of("avg")),
175182
SUM(FunctionName.of("sum")),
@@ -212,9 +219,21 @@ public enum BuiltinFunctionName {
212219
TRIM(FunctionName.of("trim")),
213220
UPPER(FunctionName.of("upper")),
214221

222+
/** Array Functions. */
223+
ARRAY(FunctionName.of("array")),
224+
215225
/** Json Functions. */
216226
JSON_VALID(FunctionName.of("json_valid")),
217227
JSON(FunctionName.of("json")),
228+
JSON_OBJECT(FunctionName.of("json_object")),
229+
JSON_ARRAY(FunctionName.of("json_array")),
230+
JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")),
231+
JSON_EXTRACT(FunctionName.of("json_extract")),
232+
JSON_KEYS(FunctionName.of("json_keys")),
233+
JSON_SET(FunctionName.of("json_set")),
234+
JSON_DELETE(FunctionName.of("json_delete")),
235+
JSON_APPEND(FunctionName.of("json_append")),
236+
JSON_EXTEND(FunctionName.of("json_extend")),
218237

219238
/** GEOSPATIAL Functions. */
220239
GEOIP(FunctionName.of("geoip")),

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
2929
import org.opensearch.sql.expression.datetime.IntervalClause;
3030
import org.opensearch.sql.expression.ip.IPFunctions;
31-
import org.opensearch.sql.expression.json.JsonFunctions;
3231
import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions;
3332
import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions;
3433
import org.opensearch.sql.expression.operator.convert.TypeCastOperators;
@@ -84,7 +83,6 @@ public static synchronized BuiltinFunctionRepository getInstance() {
8483
SystemFunctions.register(instance);
8584
OpenSearchFunctions.register(instance);
8685
IPFunctions.register(instance);
87-
JsonFunctions.register(instance);
8886
}
8987
return instance;
9088
}

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@
3131
import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils;
3232
import org.opensearch.sql.data.type.ExprCoreType;
3333
import org.opensearch.sql.expression.datetime.DateTimeFunctions;
34+
import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl;
35+
import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl;
36+
import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl;
37+
import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl;
38+
import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl;
39+
import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl;
40+
import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
41+
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
3442
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
3543
import org.opensearch.sql.expression.function.udf.GrokFunction;
3644
import org.opensearch.sql.expression.function.udf.SpanFunction;
@@ -64,6 +72,18 @@
6472
/** Defines functions and operators that are implemented only by PPL */
6573
public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
6674

75+
// Json Functions
76+
public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON");
77+
public static final SqlOperator JSON_ARRAY_LENGTH =
78+
new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH");
79+
public static final SqlOperator JSON_EXTRACT =
80+
new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT");
81+
public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS");
82+
public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET");
83+
public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE");
84+
public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND");
85+
public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND");
86+
6787
// Math functions
6888
public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN");
6989
public static final SqlOperator E = new EulerFunction().toUDF("E");

0 commit comments

Comments
 (0)