Skip to content

Commit 28d2461

Browse files
yuancuyuancu
committed
Merge remote-tracking branch 'origin/main' into cast-datetime
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
2 parents 669b461 + 774a3a2 commit 28d2461

71 files changed

Lines changed: 1626 additions & 280 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public enum SpanUnit {
4545
/** Util method to get span unit given the unit name. */
4646
public static SpanUnit of(String unit) {
4747
switch (unit) {
48+
case null:
4849
case "":
4950
return NONE;
5051
case "M":

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -320,11 +320,16 @@ public RelNode visitSort(Sort node, CalcitePlanContext context) {
320320
expr -> {
321321
RexNode sortField = rexVisitor.analyze(expr, context);
322322
SortOption sortOption = analyzeSortOption(expr.getFieldArgs());
323-
if (sortOption == DEFAULT_DESC) {
324-
return context.relBuilder.desc(sortField);
323+
// Default is ASC
324+
if (sortOption.getSortOrder() == DESC) {
325+
sortField = context.relBuilder.desc(sortField);
326+
}
327+
if (sortOption.getNullOrder() == NULL_LAST) {
328+
sortField = context.relBuilder.nullsLast(sortField);
325329
} else {
326-
return sortField;
330+
sortField = context.relBuilder.nullsFirst(sortField);
327331
}
332+
return sortField;
328333
})
329334
.collect(Collectors.toList());
330335
context.relBuilder.sort(sortList);

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java‎

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import static org.opensearch.sql.ast.expression.SpanUnit.NONE;
1212
import static org.opensearch.sql.ast.expression.SpanUnit.UNKNOWN;
1313
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY;
14+
import static org.opensearch.sql.utils.DateTimeUtils.findCastType;
15+
import static org.opensearch.sql.utils.DateTimeUtils.transferCompareForDateRelated;
1416

1517
import java.math.BigDecimal;
1618
import java.util.ArrayList;
@@ -59,6 +61,7 @@
5961
import org.opensearch.sql.ast.expression.RelevanceFieldList;
6062
import org.opensearch.sql.ast.expression.Span;
6163
import org.opensearch.sql.ast.expression.SpanUnit;
64+
import org.opensearch.sql.ast.expression.UnresolvedArgument;
6265
import org.opensearch.sql.ast.expression.UnresolvedExpression;
6366
import org.opensearch.sql.ast.expression.When;
6467
import org.opensearch.sql.ast.expression.WindowFunction;
@@ -67,15 +70,13 @@
6770
import org.opensearch.sql.ast.expression.subquery.InSubquery;
6871
import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
6972
import org.opensearch.sql.ast.tree.UnresolvedPlan;
70-
import org.opensearch.sql.calcite.type.ExprSqlType;
7173
import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
7274
import org.opensearch.sql.calcite.utils.PlanUtils;
7375
import org.opensearch.sql.common.utils.StringUtils;
7476
import org.opensearch.sql.data.type.ExprType;
7577
import org.opensearch.sql.exception.CalciteUnsupportedException;
7678
import org.opensearch.sql.exception.SemanticCheckException;
7779
import org.opensearch.sql.expression.function.BuiltinFunctionName;
78-
import org.opensearch.sql.expression.function.PPLBuiltinOperators;
7980
import org.opensearch.sql.expression.function.PPLFuncImpTable;
8081

8182
@RequiredArgsConstructor
@@ -216,28 +217,12 @@ public RexNode visitIn(In node, CalcitePlanContext context) {
216217
public RexNode visitCompare(Compare node, CalcitePlanContext context) {
217218
RexNode leftCandidate = analyze(node.getLeft(), context);
218219
RexNode rightCandidate = analyze(node.getRight(), context);
219-
Boolean whetherCompareByTime =
220-
leftCandidate.getType() instanceof ExprSqlType
221-
|| rightCandidate.getType() instanceof ExprSqlType;
222-
223-
final RexNode left =
224-
transferCompareForDateRelated(leftCandidate, context, whetherCompareByTime);
225-
final RexNode right =
226-
transferCompareForDateRelated(rightCandidate, context, whetherCompareByTime);
220+
SqlTypeName castTarget = findCastType(leftCandidate, rightCandidate);
221+
final RexNode left = transferCompareForDateRelated(leftCandidate, context, castTarget);
222+
final RexNode right = transferCompareForDateRelated(rightCandidate, context, castTarget);
227223
return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, node.getOperator(), left, right);
228224
}
229225

230-
private RexNode transferCompareForDateRelated(
231-
RexNode candidate, CalcitePlanContext context, boolean whetherCompareByTime) {
232-
if (whetherCompareByTime) {
233-
RexNode transferredStringNode =
234-
context.rexBuilder.makeCall(PPLBuiltinOperators.TIMESTAMP, candidate);
235-
return transferredStringNode;
236-
} else {
237-
return candidate;
238-
}
239-
}
240-
241226
@Override
242227
public RexNode visitBetween(Between node, CalcitePlanContext context) {
243228
RexNode value = analyze(node.getValue(), context);
@@ -669,6 +654,28 @@ public RexNode visitWhen(When node, CalcitePlanContext context) {
669654

670655
@Override
671656
public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) {
672-
throw new CalciteUnsupportedException("Relevance fields expression is unsupported in Calcite");
657+
List<RexNode> varArgRexNodeList = new ArrayList<>();
658+
node.getFieldList()
659+
.forEach(
660+
(k, v) -> {
661+
varArgRexNodeList.add(
662+
context.rexBuilder.makeLiteral(
663+
k,
664+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR),
665+
true));
666+
varArgRexNodeList.add(
667+
context.rexBuilder.makeLiteral(
668+
v,
669+
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE),
670+
true));
671+
});
672+
return context.rexBuilder.makeCall(
673+
SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR, varArgRexNodeList);
674+
}
675+
676+
@Override
677+
public RexNode visitUnresolvedArgument(UnresolvedArgument node, CalcitePlanContext context) {
678+
RexNode value = analyze(node.getValue(), context);
679+
return context.relBuilder.alias(value, node.getArgName());
673680
}
674681
}

‎core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.*;
1111
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.*;
1212

13+
import com.google.common.collect.ImmutableSet;
1314
import java.time.Instant;
1415
import java.time.ZoneId;
1516
import java.util.ArrayList;
1617
import java.util.Collections;
1718
import java.util.List;
19+
import java.util.Set;
1820
import java.util.TimeZone;
1921
import javax.annotation.Nullable;
2022
import org.apache.calcite.DataContext;
@@ -70,6 +72,10 @@ public class UserDefinedFunctionUtils {
7072
TYPE_FACTORY.createMapType(
7173
TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR),
7274
createArrayType(TYPE_FACTORY, TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), false));
75+
public static Set<String> SINGLE_FIELD_RELEVANCE_FUNCTION_SET =
76+
ImmutableSet.of("match", "match_phrase", "match_bool_prefix", "match_phrase_prefix");
77+
public static Set<String> MULTI_FIELDS_RELEVANCE_FUNCTION_SET =
78+
ImmutableSet.of("simple_query_string", "query_string", "multi_match");
7379

7480
public static RelBuilder.AggCall TransferUserDefinedAggFunction(
7581
Class<? extends UserDefinedAggFunction> UDAF,

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
4848
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
4949
import org.opensearch.sql.expression.function.udf.GrokFunction;
50+
import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction;
5051
import org.opensearch.sql.expression.function.udf.SpanFunction;
5152
import org.opensearch.sql.expression.function.udf.condition.EarliestFunction;
5253
import org.opensearch.sql.expression.function.udf.condition.LatestFunction;
@@ -327,6 +328,22 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
327328
public static final SqlOperator TRANSFORM = new TransformFunctionImpl().toUDF("transform");
328329
public static final SqlOperator REDUCE = new ReduceFunctionImpl().toUDF("reduce");
329330

331+
private static final RelevanceQueryFunction RELEVANCE_QUERY_FUNCTION_INSTANCE =
332+
new RelevanceQueryFunction();
333+
public static final SqlOperator MATCH = RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match");
334+
public static final SqlOperator MATCH_PHRASE =
335+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase");
336+
public static final SqlOperator MATCH_BOOL_PREFIX =
337+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_bool_prefix");
338+
public static final SqlOperator MATCH_PHRASE_PREFIX =
339+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("match_phrase_prefix");
340+
public static final SqlOperator SIMPLE_QUERY_STRING =
341+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("simple_query_string");
342+
public static final SqlOperator QUERY_STRING =
343+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("query_string");
344+
public static final SqlOperator MULTI_MATCH =
345+
RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match");
346+
330347
/**
331348
* Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're
332349
* all private Use method directly in {@link BuiltInMethod} if possible, most operators'

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@
119119
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM;
120120
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE;
121121
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME;
122+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH;
123+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_BOOL_PREFIX;
124+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE;
125+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE_PREFIX;
122126
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAX;
123127
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5;
124128
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND;
@@ -133,6 +137,7 @@
133137
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME;
134138
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR;
135139
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY;
140+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH;
136141
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT;
137142
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL;
138143
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW;
@@ -146,6 +151,7 @@
146151
import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW;
147152
import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER;
148153
import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUARTER;
154+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUERY_STRING;
149155
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS;
150156
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND;
151157
import static org.opensearch.sql.expression.function.BuiltinFunctionName.REDUCE;
@@ -161,6 +167,7 @@
161167
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1;
162168
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2;
163169
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN;
170+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIMPLE_QUERY_STRING;
164171
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN;
165172
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN;
166173
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SQRT;
@@ -693,6 +700,13 @@ void populate() {
693700
registerOperator(SHA2, PPLBuiltinOperators.SHA2);
694701
registerOperator(CIDRMATCH, PPLBuiltinOperators.CIDRMATCH);
695702
registerOperator(INTERNAL_GROK, PPLBuiltinOperators.GROK);
703+
registerOperator(MATCH, PPLBuiltinOperators.MATCH);
704+
registerOperator(MATCH_PHRASE, PPLBuiltinOperators.MATCH_PHRASE);
705+
registerOperator(MATCH_BOOL_PREFIX, PPLBuiltinOperators.MATCH_BOOL_PREFIX);
706+
registerOperator(MATCH_PHRASE_PREFIX, PPLBuiltinOperators.MATCH_PHRASE_PREFIX);
707+
registerOperator(SIMPLE_QUERY_STRING, PPLBuiltinOperators.SIMPLE_QUERY_STRING);
708+
registerOperator(QUERY_STRING, PPLBuiltinOperators.QUERY_STRING);
709+
registerOperator(MULTI_MATCH, PPLBuiltinOperators.MULTI_MATCH);
696710

697711
// Register PPL Datetime UDF operator
698712
registerOperator(TIMESTAMP, PPLBuiltinOperators.TIMESTAMP);

‎core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import com.google.common.collect.ImmutableList;
9+
import java.util.List;
10+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
11+
import org.apache.calcite.adapter.enumerable.NullPolicy;
12+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
13+
import org.apache.calcite.linq4j.tree.Expression;
14+
import org.apache.calcite.rex.RexCall;
15+
import org.apache.calcite.sql.type.CompositeOperandTypeChecker;
16+
import org.apache.calcite.sql.type.OperandTypes;
17+
import org.apache.calcite.sql.type.ReturnTypes;
18+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
19+
import org.apache.calcite.sql.type.SqlTypeFamily;
20+
import org.opensearch.sql.expression.function.ImplementorUDF;
21+
import org.opensearch.sql.expression.function.UDFOperandMetadata;
22+
23+
public class RelevanceQueryFunction extends ImplementorUDF {
24+
25+
public RelevanceQueryFunction() {
26+
super(new RelevanceQueryImplementor(), NullPolicy.ANY);
27+
}
28+
29+
@Override
30+
public SqlReturnTypeInference getReturnTypeInference() {
31+
return ReturnTypes.BOOLEAN;
32+
}
33+
34+
/*
35+
* Starting from the 3rd parameter, they are optional parameters for relevance queries.
36+
* Different query has different parameter set, which will be validated in dedicated query builder
37+
*/
38+
@Override
39+
public UDFOperandMetadata getOperandMetadata() {
40+
return UDFOperandMetadata.wrap(
41+
(CompositeOperandTypeChecker)
42+
OperandTypes.family(
43+
ImmutableList.of(
44+
SqlTypeFamily.STRING,
45+
SqlTypeFamily.STRING,
46+
SqlTypeFamily.STRING,
47+
SqlTypeFamily.STRING,
48+
SqlTypeFamily.STRING,
49+
SqlTypeFamily.STRING,
50+
SqlTypeFamily.STRING,
51+
SqlTypeFamily.STRING,
52+
SqlTypeFamily.STRING,
53+
SqlTypeFamily.STRING,
54+
SqlTypeFamily.STRING,
55+
SqlTypeFamily.STRING,
56+
SqlTypeFamily.STRING,
57+
SqlTypeFamily.STRING),
58+
i -> i > 1 && i < 14) // Parameters 3-14 are optional
59+
.or(
60+
OperandTypes.family(
61+
ImmutableList.of(
62+
SqlTypeFamily.MAP,
63+
SqlTypeFamily.STRING,
64+
SqlTypeFamily.STRING,
65+
SqlTypeFamily.STRING,
66+
SqlTypeFamily.STRING,
67+
SqlTypeFamily.STRING,
68+
SqlTypeFamily.STRING,
69+
SqlTypeFamily.STRING,
70+
SqlTypeFamily.STRING,
71+
SqlTypeFamily.STRING,
72+
SqlTypeFamily.STRING,
73+
SqlTypeFamily.STRING,
74+
SqlTypeFamily.STRING,
75+
SqlTypeFamily.STRING,
76+
SqlTypeFamily.STRING,
77+
SqlTypeFamily.STRING,
78+
SqlTypeFamily.STRING,
79+
SqlTypeFamily.STRING,
80+
SqlTypeFamily.STRING,
81+
SqlTypeFamily.STRING,
82+
SqlTypeFamily.STRING,
83+
SqlTypeFamily.STRING,
84+
SqlTypeFamily.STRING,
85+
SqlTypeFamily.STRING,
86+
SqlTypeFamily.STRING),
87+
i -> i > 1 && i < 25))); // Parameters 3-25 are optional
88+
}
89+
90+
public static class RelevanceQueryImplementor implements NotNullImplementor {
91+
@Override
92+
public Expression implement(
93+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
94+
throw new UnsupportedOperationException(
95+
"Relevance search query functions are only supported when they are pushed down");
96+
}
97+
}
98+
}

‎core/src/main/java/org/opensearch/sql/expression/function/udf/SpanFunction.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,14 @@ public SqlReturnTypeInference getReturnTypeInference() {
5151
public UDFOperandMetadata getOperandMetadata() {
5252
return UDFOperandMetadata.wrap(
5353
(CompositeOperandTypeChecker)
54-
OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.INTEGER, SqlTypeFamily.STRING)
54+
OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)
5555
.or(
5656
OperandTypes.family(
57-
SqlTypeFamily.DATETIME, SqlTypeFamily.INTEGER, SqlTypeFamily.STRING))
57+
SqlTypeFamily.DATETIME, SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING))
58+
// TODO: numeric span should support decimal as its interval
5859
.or(
5960
OperandTypes.family(
60-
SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER, SqlTypeFamily.ANY)));
61+
SqlTypeFamily.NUMERIC, SqlTypeFamily.INTEGER, SqlTypeFamily.ANY)));
6162
}
6263

6364
public static class SpanImplementor implements NotNullImplementor {

0 commit comments

Comments
 (0)