Skip to content

Commit 7ab3520

Browse files
yuancuyuancu
committed
Merge origin/main into explain-no-pushdown
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
2 parents a874386 + 774a3a2 commit 7ab3520

31 files changed

Lines changed: 1025 additions & 82 deletions

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import org.opensearch.sql.ast.expression.RelevanceFieldList;
6262
import org.opensearch.sql.ast.expression.Span;
6363
import org.opensearch.sql.ast.expression.SpanUnit;
64+
import org.opensearch.sql.ast.expression.UnresolvedArgument;
6465
import org.opensearch.sql.ast.expression.UnresolvedExpression;
6566
import org.opensearch.sql.ast.expression.When;
6667
import org.opensearch.sql.ast.expression.WindowFunction;
@@ -653,6 +654,28 @@ public RexNode visitWhen(When node, CalcitePlanContext context) {
653654

654655
@Override
655656
public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) {
656-
throw new CalciteUnsupportedException("Relevance fields expression is unsupported in Calcite");
657+
List<RexNode> varArgRexNodeList = new ArrayList<>();
658+
node.getFieldList()
659+
.forEach(
660+
(k, v) -> {
661+
varArgRexNodeList.add(
662+
context.rexBuilder.makeLiteral(
663+
k,
664+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR),
665+
true));
666+
varArgRexNodeList.add(
667+
context.rexBuilder.makeLiteral(
668+
v,
669+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE),
670+
true));
671+
});
672+
return context.rexBuilder.makeCall(
673+
SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, varArgRexNodeList);
674+
}
675+
676+
@Override
677+
public RexNode visitUnresolvedArgument(UnresolvedArgument node, CalcitePlanContext context) {
678+
RexNode value = analyze(node.getValue(), context);
679+
return context.relBuilder.alias(value, node.getArgName());
657680
}
658681
}

‎core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.*;
1111
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.*;
1212

13+
import com.google.common.collect.ImmutableSet;
1314
import java.time.Instant;
1415
import java.time.ZoneId;
1516
import java.util.ArrayList;
1617
import java.util.Collections;
1718
import java.util.List;
19+
import java.util.Set;
1820
import java.util.TimeZone;
1921
import javax.annotation.Nullable;
2022
import org.apache.calcite.DataContext;
@@ -70,6 +72,10 @@ public class UserDefinedFunctionUtils {
7072
TYPE_FACTORY.createMapType(
7173
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
7274
createArrayType(TYPE_FACTORY, TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), false));
75+
public static Set<String> SINGLE_FIELD_RELEVANCE_FUNCTION_SET =
76+
ImmutableSet.of("match", "match_phrase", "match_bool_prefix", "match_phrase_prefix");
77+
public static Set<String> MULTI_FIELDS_RELEVANCE_FUNCTION_SET =
78+
ImmutableSet.of("simple_query_string", "query_string", "multi_match");
7379

7480
public static RelBuilder.AggCall TransferUserDefinedAggFunction(
7581
Class<? extends UserDefinedAggFunction> UDAF,

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
4848
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
4949
import org.opensearch.sql.expression.function.udf.GrokFunction;
50+
import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction;
5051
import org.opensearch.sql.expression.function.udf.SpanFunction;
5152
import org.opensearch.sql.expression.function.udf.condition.EarliestFunction;
5253
import org.opensearch.sql.expression.function.udf.condition.LatestFunction;
@@ -327,6 +328,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
327328
public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform");
328329
public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce");
329330

331+
private static final RelevanceQueryFunction RELEVANCE_QUERY_FUNCTION_INSTANCE =
332+
new RelevanceQueryFunction();
333+
public static final SqlOperator MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match");
334+
public static final SqlOperator MATCH_PHRASE =
335+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase");
336+
public static final SqlOperator MATCH_BOOL_PREFIX =
337+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_bool_prefix");
338+
public static final SqlOperator MATCH_PHRASE_PREFIX =
339+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase_prefix");
340+
public static final SqlOperator SIMPLE_QUERY_STRING =
341+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("simple_query_string");
342+
public static final SqlOperator QUERY_STRING =
343+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("query_string");
344+
public static final SqlOperator MULTI_MATCH =
345+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match");
346+
330347
/**
331348
* Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're
332349
* all private Use method directly in {@link BuiltInMethod} if possible, most operators'

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@
119119
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM;
120120
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE;
121121
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME;
122+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH;
123+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_BOOL_PREFIX;
124+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE;
125+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE_PREFIX;
122126
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAX;
123127
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5;
124128
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND;
@@ -133,6 +137,7 @@
133137
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME;
134138
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR;
135139
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY;
140+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH;
136141
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT;
137142
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL;
138143
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW;
@@ -146,6 +151,7 @@
146151
import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW;
147152
import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER;
148153
import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUARTER;
154+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUERY_STRING;
149155
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS;
150156
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND;
151157
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REDUCE;
@@ -161,6 +167,7 @@
161167
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1;
162168
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2;
163169
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN;
170+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIMPLE_QUERY_STRING;
164171
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN;
165172
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN;
166173
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SQRT;
@@ -693,6 +700,13 @@ void populate() {
693700
registerOperator(SHA2, PPLBuiltinOperators.SHA2);
694701
registerOperator(CIDRMATCH, PPLBuiltinOperators.CIDRMATCH);
695702
registerOperator(INTERNAL_GROK, PPLBuiltinOperators.GROK);
703+
registerOperator(MATCH, PPLBuiltinOperators.MATCH);
704+
registerOperator(MATCH_PHRASE, PPLBuiltinOperators.MATCH_PHRASE);
705+
registerOperator(MATCH_BOOL_PREFIX, PPLBuiltinOperators.MATCH_BOOL_PREFIX);
706+
registerOperator(MATCH_PHRASE_PREFIX, PPLBuiltinOperators.MATCH_PHRASE_PREFIX);
707+
registerOperator(SIMPLE_QUERY_STRING, PPLBuiltinOperators.SIMPLE_QUERY_STRING);
708+
registerOperator(QUERY_STRING, PPLBuiltinOperators.QUERY_STRING);
709+
registerOperator(MULTI_MATCH, PPLBuiltinOperators.MULTI_MATCH);
696710

697711
// Register PPL Datetime UDF operator
698712
registerOperator(TIMESTAMP, PPLBuiltinOperators.TIMESTAMP);

‎core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import com.google.common.collect.ImmutableList;
9+
import java.util.List;
10+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
11+
import org.apache.calcite.adapter.enumerable.NullPolicy;
12+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
13+
import org.apache.calcite.linq4j.tree.Expression;
14+
import org.apache.calcite.rex.RexCall;
15+
import org.apache.calcite.sql.type.CompositeOperandTypeChecker;
16+
import org.apache.calcite.sql.type.OperandTypes;
17+
import org.apache.calcite.sql.type.ReturnTypes;
18+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
19+
import org.apache.calcite.sql.type.SqlTypeFamily;
20+
import org.opensearch.sql.expression.function.ImplementorUDF;
21+
import org.opensearch.sql.expression.function.UDFOperandMetadata;
22+
23+
public class RelevanceQueryFunction extends ImplementorUDF {
24+
25+
public RelevanceQueryFunction() {
26+
super(new RelevanceQueryImplementor(), NullPolicy.ANY);
27+
}
28+
29+
@Override
30+
public SqlReturnTypeInference getReturnTypeInference() {
31+
return ReturnTypes.BOOLEAN;
32+
}
33+
34+
/*
35+
* Starting from the 3rd parameter, they are optional parameters for relevance queries.
36+
* Different query has different parameter set, which will be validated in dedicated query builder
37+
*/
38+
@Override
39+
public UDFOperandMetadata getOperandMetadata() {
40+
return UDFOperandMetadata.wrap(
41+
(CompositeOperandTypeChecker)
42+
OperandTypes.family(
43+
ImmutableList.of(
44+
SqlTypeFamily.STRING,
45+
SqlTypeFamily.STRING,
46+
SqlTypeFamily.STRING,
47+
SqlTypeFamily.STRING,
48+
SqlTypeFamily.STRING,
49+
SqlTypeFamily.STRING,
50+
SqlTypeFamily.STRING,
51+
SqlTypeFamily.STRING,
52+
SqlTypeFamily.STRING,
53+
SqlTypeFamily.STRING,
54+
SqlTypeFamily.STRING,
55+
SqlTypeFamily.STRING,
56+
SqlTypeFamily.STRING,
57+
SqlTypeFamily.STRING),
58+
i -> i > 1 && i < 14) // Parameters 3-14 are optional
59+
.or(
60+
OperandTypes.family(
61+
ImmutableList.of(
62+
SqlTypeFamily.MAP,
63+
SqlTypeFamily.STRING,
64+
SqlTypeFamily.STRING,
65+
SqlTypeFamily.STRING,
66+
SqlTypeFamily.STRING,
67+
SqlTypeFamily.STRING,
68+
SqlTypeFamily.STRING,
69+
SqlTypeFamily.STRING,
70+
SqlTypeFamily.STRING,
71+
SqlTypeFamily.STRING,
72+
SqlTypeFamily.STRING,
73+
SqlTypeFamily.STRING,
74+
SqlTypeFamily.STRING,
75+
SqlTypeFamily.STRING,
76+
SqlTypeFamily.STRING,
77+
SqlTypeFamily.STRING,
78+
SqlTypeFamily.STRING,
79+
SqlTypeFamily.STRING,
80+
SqlTypeFamily.STRING,
81+
SqlTypeFamily.STRING,
82+
SqlTypeFamily.STRING,
83+
SqlTypeFamily.STRING,
84+
SqlTypeFamily.STRING,
85+
SqlTypeFamily.STRING,
86+
SqlTypeFamily.STRING),
87+
i -> i > 1 && i < 25))); // Parameters 3-25 are optional
88+
}
89+
90+
public static class RelevanceQueryImplementor implements NotNullImplementor {
91+
@Override
92+
public Expression implement(
93+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
94+
throw new UnsupportedOperationException(
95+
"Relevance search query functions are only supported when they are pushed down");
96+
}
97+
}
98+
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,6 @@
4646
CalciteJsonFunctionsIT.class,
4747
CalciteLegacyAPICompatibilityIT.class,
4848
CalciteLikeQueryIT.class,
49-
CalciteMatchBoolPrefixIT.class,
50-
CalciteMatchIT.class,
51-
CalciteMatchPhraseIT.class,
5249
CalciteMathematicalFunctionIT.class,
5350
CalciteNewAddedCommandsIT.class,
5451
CalciteNowLikeFunctionIT.class,
@@ -86,15 +83,12 @@
8683
CalcitePPLTrendlineIT.class,
8784
CalcitePrometheusDataSourceCommandsIT.class,
8885
CalciteQueryAnalysisIT.class,
89-
CalciteQueryStringIT.class,
9086
CalciteRareCommandIT.class,
91-
CalciteRelevanceFunctionIT.class,
9287
CalciteRenameCommandIT.class,
9388
CalciteResourceMonitorIT.class,
9489
CalciteSearchCommandIT.class,
9590
CalciteSettingsIT.class,
9691
CalciteShowDataSourcesCommandIT.class,
97-
CalciteSimpleQueryStringIT.class,
9892
CalciteSortCommandIT.class,
9993
CalciteStatsCommandIT.class,
10094
CalciteSystemFunctionIT.class,

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55

66
package org.opensearch.sql.calcite.remote;
77

8+
import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId;
9+
810
import java.io.IOException;
911
import org.junit.Ignore;
12+
import org.junit.Test;
1013
import org.opensearch.sql.ppl.ExplainIT;
1114

1215
public class CalciteExplainIT extends ExplainIT {
@@ -21,8 +24,39 @@ public void init() throws Exception {
2124
@Ignore("test only in v2")
2225
public void testExplainModeUnsupportedInV2() throws IOException {}
2326

24-
@Override
25-
public void testExplain() throws IOException {
26-
super.testExplain();
27+
// Only for Calcite
28+
@Test
29+
public void supportSearchSargPushDown_singleRange() throws IOException {
30+
String query =
31+
"source=opensearch-sql_test_index_account | where age >= 1.0 and age < 10 | fields age";
32+
var result = explainQueryToString(query);
33+
String expected =
34+
loadFromFile("expectedOutput/calcite/explain_sarg_filter_push_single_range.json");
35+
assertJsonEqualsIgnoreId(expected, result);
36+
}
37+
38+
// Only for Calcite
39+
@Test
40+
public void supportSearchSargPushDown_multiRange() throws IOException {
41+
String query =
42+
"source=opensearch-sql_test_index_account | where (age > 20 and age < 28) or (age > 25 and"
43+
+ " age < 30) or (age >= 1 and age <= 10) or age = 0 | fields age";
44+
var result = explainQueryToString(query);
45+
String expected =
46+
loadFromFile("expectedOutput/calcite/explain_sarg_filter_push_multi_range.json");
47+
assertJsonEqualsIgnoreId(expected, result);
48+
}
49+
50+
// Only for Calcite
51+
@Test
52+
public void supportSearchSargPushDown_timeRange() throws IOException {
53+
String expected =
54+
loadFromFile("expectedOutput/calcite/explain_sarg_filter_push_time_range.json");
55+
assertJsonEqualsIgnoreId(
56+
expected,
57+
explainQueryToString(
58+
"source=opensearch-sql_test_index_bank"
59+
+ "| where birthdate >= '2016-12-08 00:00:00.000000000' "
60+
+ "and birthdate < '2018-11-09 00:00:00.000000000' "));
2761
}
2862
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMatchBoolPrefixIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ public class CalciteMatchBoolPrefixIT extends MatchBoolPrefixIT {
1212
public void init() throws Exception {
1313
super.init();
1414
enableCalcite();
15-
// TODO: "https://github.com/opensearch-project/sql/issues/3462"
16-
// disallowCalciteFallback();
15+
disallowCalciteFallback();
1716
}
1817
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMatchIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ public class CalciteMatchIT extends MatchIT {
1212
public void init() throws Exception {
1313
super.init();
1414
enableCalcite();
15-
// TODO: "https://github.com/opensearch-project/sql/issues/3462"
16-
// disallowCalciteFallback();
15+
disallowCalciteFallback();
1716
}
1817
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMatchPhraseIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ public class CalciteMatchPhraseIT extends MatchPhraseIT {
1212
public void init() throws Exception {
1313
super.init();
1414
enableCalcite();
15-
// TODO: "https://github.com/opensearch-project/sql/issues/3462"
16-
// disallowCalciteFallback();
15+
disallowCalciteFallback();
1716
}
1817
}

0 commit comments

Comments
 (0)