Skip to content

Commit 2b8ba89

Browse files
LantaoJinLantaoJin
authored
Merge branch '2.19-dev' into backport/backport-3901-to-2.19-dev
Signed-off-by: Lantao Jin <ltjin@amazon.com>
2 parents 9bbb77b + 837f97d commit 2b8ba89

229 files changed

Lines changed: 5943 additions & 1962 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/ExpressionScriptSerdeBenchmark.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.operator.predicate;
7+
8+
import java.util.Map;
9+
import java.util.concurrent.TimeUnit;
10+
import org.apache.calcite.plan.RelOptCluster;
11+
import org.apache.calcite.plan.volcano.VolcanoPlanner;
12+
import org.apache.calcite.rel.type.RelDataType;
13+
import org.apache.calcite.rel.type.StructKind;
14+
import org.apache.calcite.rex.RexBuilder;
15+
import org.apache.calcite.rex.RexNode;
16+
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
17+
import org.apache.calcite.sql.type.SqlTypeName;
18+
import org.openjdk.jmh.annotations.Benchmark;
19+
import org.openjdk.jmh.annotations.BenchmarkMode;
20+
import org.openjdk.jmh.annotations.Fork;
21+
import org.openjdk.jmh.annotations.Measurement;
22+
import org.openjdk.jmh.annotations.Mode;
23+
import org.openjdk.jmh.annotations.OutputTimeUnit;
24+
import org.openjdk.jmh.annotations.Scope;
25+
import org.openjdk.jmh.annotations.State;
26+
import org.openjdk.jmh.annotations.Warmup;
27+
import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
28+
import org.opensearch.sql.data.type.ExprCoreType;
29+
import org.opensearch.sql.data.type.ExprType;
30+
import org.opensearch.sql.expression.DSL;
31+
import org.opensearch.sql.expression.Expression;
32+
import org.opensearch.sql.expression.function.BuiltinFunctionName;
33+
import org.opensearch.sql.expression.function.PPLFuncImpTable;
34+
import org.opensearch.sql.opensearch.storage.serde.DefaultExpressionSerializer;
35+
import org.opensearch.sql.opensearch.storage.serde.RelJsonSerializer;
36+
37+
@Warmup(iterations = 1)
38+
@Measurement(iterations = 10)
39+
@BenchmarkMode(Mode.AverageTime)
40+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
41+
@State(Scope.Thread)
42+
@Fork(value = 1)
43+
public class ExpressionScriptSerdeBenchmark {
44+
45+
@Benchmark
46+
public void testV2ExpressionSerde() {
47+
DefaultExpressionSerializer defaultSerializer = new DefaultExpressionSerializer();
48+
Expression exprUpper = DSL.upper(DSL.ref("Referer", ExprCoreType.STRING));
49+
Expression exprNotEquals = DSL.notequal(exprUpper, DSL.literal("ABOUT"));
50+
51+
String serializedStr = defaultSerializer.serialize(exprNotEquals);
52+
defaultSerializer.deserialize(serializedStr);
53+
}
54+
55+
@Benchmark
56+
public void testRexNodeJsonSerde() {
57+
RexBuilder rexBuilder = new RexBuilder(OpenSearchTypeFactory.TYPE_FACTORY);
58+
RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder);
59+
RelJsonSerializer relJsonSerializer = new RelJsonSerializer(cluster);
60+
RelDataType rowType =
61+
rexBuilder
62+
.getTypeFactory()
63+
.builder()
64+
.kind(StructKind.FULLY_QUALIFIED)
65+
.add("Referer", rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR))
66+
.build();
67+
RexNode rexUpper =
68+
PPLFuncImpTable.INSTANCE.resolve(
69+
rexBuilder,
70+
BuiltinFunctionName.UPPER,
71+
rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0));
72+
RexNode rexNotEquals =
73+
rexBuilder.makeCall(
74+
SqlStdOperatorTable.NOT_EQUALS, rexUpper, rexBuilder.makeLiteral("ABOUT"));
75+
Map<String, ExprType> fieldTypes = Map.of("Referer", ExprCoreType.STRING);
76+
77+
String serializedStr = relJsonSerializer.serialize(rexNotEquals, rowType, fieldTypes);
78+
relJsonSerializer.deserialize(serializedStr);
79+
}
80+
}

‎benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import java.util.Map;
1313
import org.openjdk.jmh.annotations.Benchmark;
1414
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
15-
import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
15+
import org.opensearch.sql.opensearch.util.MergeRules.MergeRuleHelper;
1616

1717
public class MergeArrayAndObjectMapBenchmark {
1818
private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
@@ -21,7 +21,7 @@ public class MergeArrayAndObjectMapBenchmark {
2121
public void testMerge() {
2222
Map<String, OpenSearchDataType> finalResult = new HashMap<>();
2323
for (Map<String, OpenSearchDataType> map : candidateMaps) {
24-
OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
24+
MergeRuleHelper.merge(finalResult, map);
2525
}
2626
}
2727

‎build.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ buildscript {
5656
mockito_version = "5.7.0"
5757
commons_io_version = "2.14.0"
5858
commons_text_version = "1.10.0"
59-
commons_lang3_version = "3.12.0"
59+
commons_lang3_version = "3.18.0"
6060
// enforce 1.13, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379
6161
commons_codec_version = "1.13"
6262
commons_logging_version = "1.2"
@@ -145,7 +145,7 @@ allprojects {
145145
resolutionStrategy.force 'org.locationtech.jts:jts-core:1.19.0'
146146
resolutionStrategy.force 'com.google.errorprone:error_prone_annotations:2.28.0'
147147
resolutionStrategy.force 'org.checkerframework:checker-qual:3.43.0'
148-
resolutionStrategy.force 'org.apache.commons:commons-lang3:3.13.0'
148+
resolutionStrategy.force 'org.apache.commons:commons-lang3:3.18.0'
149149
resolutionStrategy.force 'org.apache.commons:commons-text:1.11.0'
150150
resolutionStrategy.force 'commons-io:commons-io:2.15.0'
151151
resolutionStrategy.force 'org.yaml:snakeyaml:2.2'

‎common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public Grok compile(String pattern) throws IllegalArgumentException {
117117
}
118118

119119
public Grok compile(final String pattern, boolean namedOnly) throws IllegalArgumentException {
120-
return compile(pattern, ZoneOffset.systemDefault(), namedOnly);
120+
return compile(pattern, ZoneOffset.UTC, namedOnly);
121121
}
122122

123123
/**

‎common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -680,8 +680,7 @@ public void testTimeZone() {
680680
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss");
681681
Grok grok = compiler.compile("%{DATESTAMP:timestamp;date;MM/dd/yyyy HH:mm:ss}", true);
682682
Instant instant = (Instant) grok.match(date).capture().get("timestamp");
683-
assertEquals(
684-
ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.systemDefault())).toInstant(), instant);
683+
assertEquals(ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.UTC)).toInstant(), instant);
685684

686685
// set default timezone to PST
687686
ZoneId pst = ZoneId.of("PST", ZoneId.SHORT_IDS);

‎core/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ dependencies {
4848
api "net.minidev:json-smart:2.5.2"
4949
api('org.apache.calcite:calcite-core:1.38.0') {
5050
exclude group: 'net.minidev', module: 'json-smart'
51+
exclude group: 'commons-lang', module: 'commons-lang'
5152
}
5253
api 'org.apache.calcite:calcite-linq4j:1.38.0'
5354
api project(':common')

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,11 @@ private Pair<List<RexNode>, List<AggCall>> aggregateWithTrimming(
560560
// \- Project([c, b])
561561
// \- Filter(a > 1)
562562
// \- Scan t
563+
// Example 3: source=t | stats count(): no project added for count()
564+
// Before: Aggregate(count)
565+
// \- Scan t
566+
// After: Aggregate(count)
567+
// \- Scan t
563568
Pair<List<RexNode>, List<AggCall>> resolved =
564569
resolveAttributesForAggregation(groupExprList, aggExprList, context);
565570
List<RexInputRef> trimmedRefs = new ArrayList<>();

‎core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import java.util.List;
1111
import java.util.stream.Collectors;
1212

13+
import java.util.Locale;
1314
import org.apache.calcite.avatica.util.TimeUnit;
1415
import org.apache.calcite.rel.type.RelDataType;
1516
import org.apache.calcite.rex.RexBuilder;
@@ -21,6 +22,12 @@
2122
import org.apache.calcite.sql.type.SqlTypeName;
2223
import org.apache.calcite.sql.type.SqlTypeUtil;
2324
import org.opensearch.sql.ast.expression.SpanUnit;
25+
import org.opensearch.sql.calcite.type.AbstractExprRelDataType;
26+
import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
27+
import org.opensearch.sql.data.type.ExprCoreType;
28+
import org.opensearch.sql.exception.ExpressionEvaluationException;
29+
import org.opensearch.sql.exception.SemanticCheckException;
30+
import org.opensearch.sql.expression.function.PPLBuiltinOperators;
2431

2532
public class ExtendedRexBuilder extends RexBuilder {
2633

@@ -119,7 +126,35 @@ public RexNode makeCast(
119126
// SqlStdOperatorTable.NOT_EQUALS,
120127
// ImmutableList.of(exp, makeZeroLiteral(exp.getType())));
121128
}
122-
}
129+
} else if (OpenSearchTypeFactory.isUserDefinedType(type)) {
130+
var udt = ((AbstractExprRelDataType<?>) type).getUdt();
131+
var argExprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(exp.getType());
132+
switch (udt) {
133+
case EXPR_DATE:
134+
return makeCall(type, PPLBuiltinOperators.DATE, List.of(exp));
135+
case EXPR_TIME:
136+
return makeCall(type, PPLBuiltinOperators.TIME, List.of(exp));
137+
case EXPR_TIMESTAMP:
138+
return makeCall(type, PPLBuiltinOperators.TIMESTAMP, List.of(exp));
139+
case EXPR_IP:
140+
if (argExprType == ExprCoreType.IP) {
141+
return exp;
142+
} else if (argExprType == ExprCoreType.STRING) {
143+
return makeCall(type, PPLBuiltinOperators.IP, List.of(exp));
144+
}
145+
// Throwing error inside implementation will be suppressed by Calcite, thus
146+
// throwing 500 error. Therefore, we throw error here to ensure the error
147+
// information is displayed properly.
148+
throw new ExpressionEvaluationException(
149+
String.format(
150+
Locale.ROOT,
151+
"Cannot convert %s to IP, only STRING and IP types are supported",
152+
argExprType));
153+
default:
154+
throw new SemanticCheckException(
155+
String.format(Locale.ROOT, "Cannot cast from %s to %s", argExprType, udt.name()));
156+
}
157+
}
123158
return super.makeCast(pos, type, exp, matchNullability, safe, format);
124159
}
125160
}

‎core/src/main/java/org/opensearch/sql/calcite/plan/Scannable.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@
1616
*/
1717
public interface Scannable {
1818

19-
public Enumerable<@Nullable Object> scan();
19+
public Enumerable<@Nullable Object> scanWithLimit();
2020
}

‎core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ protected PreparedResult implement(RelRoot root) {
302302
RelDataType resultType = root.rel.getRowType();
303303
boolean isDml = root.kind.belongsTo(SqlKind.DML);
304304
if (root.rel instanceof Scannable) {
305-
final Bindable bindable = dataContext -> ((Scannable) root.rel).scan();
305+
final Bindable bindable = dataContext -> ((Scannable) root.rel).scanWithLimit();
306306

307307
return new PreparedResultImpl(
308308
resultType,

0 commit comments

Comments
 (0)