Skip to content

Commit 9f74cca

Browse files
xinyualxinyual
committed
cast to time/date when meeting literal
Signed-off-by: xinyual <xinyual@amazon.com>
1 parent 1dd6ebc commit 9f74cca

7 files changed

Lines changed: 169 additions & 19 deletions

File tree

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@
1010
import static org.apache.commons.lang3.StringUtils.substringAfterLast;
1111
import static org.opensearch.sql.ast.expression.SpanUnit.NONE;
1212
import static org.opensearch.sql.ast.expression.SpanUnit.UNKNOWN;
13+
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_DATE;
14+
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIME;
1315
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP;
1416
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY;
17+
import static org.opensearch.sql.utils.DateTimeUtils.findCastType;
18+
import static org.opensearch.sql.utils.DateTimeUtils.transferCompareForDateRelated;
1519

1620
import java.math.BigDecimal;
1721
import java.util.ArrayList;
@@ -20,6 +24,7 @@
2024
import java.util.List;
2125
import java.util.Locale;
2226
import java.util.Map;
27+
import java.util.Objects;
2328
import java.util.stream.Collectors;
2429
import java.util.stream.IntStream;
2530
import javax.annotation.Nullable;
@@ -217,30 +222,14 @@ public RexNode visitIn(In node, CalcitePlanContext context) {
217222
public RexNode visitCompare(Compare node, CalcitePlanContext context) {
218223
RexNode leftCandidate = analyze(node.getLeft(), context);
219224
RexNode rightCandidate = analyze(node.getRight(), context);
220-
Boolean whetherCompareByTime =
221-
leftCandidate.getType() instanceof ExprSqlType
222-
|| rightCandidate.getType() instanceof ExprSqlType;
223-
225+
SqlTypeName castTarget = findCastType(leftCandidate, rightCandidate);
224226
final RexNode left =
225-
transferCompareForDateRelated(leftCandidate, context, whetherCompareByTime);
227+
transferCompareForDateRelated(leftCandidate, context, castTarget);
226228
final RexNode right =
227-
transferCompareForDateRelated(rightCandidate, context, whetherCompareByTime);
229+
transferCompareForDateRelated(rightCandidate, context, castTarget);
228230
return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, node.getOperator(), left, right);
229231
}
230232

231-
private RexNode transferCompareForDateRelated(
232-
RexNode candidate, CalcitePlanContext context, boolean whetherCompareByTime) {
233-
if (whetherCompareByTime) {
234-
if (!(candidate.getType() instanceof ExprSqlType
235-
&& ((ExprSqlType) candidate.getType()).getUdt() == EXPR_TIMESTAMP)) {
236-
RexNode transferredStringNode =
237-
context.rexBuilder.makeCall(PPLBuiltinOperators.TIMESTAMP, candidate);
238-
return transferredStringNode;
239-
}
240-
}
241-
return candidate;
242-
}
243-
244233
@Override
245234
public RexNode visitBetween(Between node, CalcitePlanContext context) {
246235
RexNode value = analyze(node.getValue(), context);

‎core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,22 @@
1515
import java.time.format.DateTimeParseException;
1616
import java.time.temporal.ChronoUnit;
1717
import java.util.Locale;
18+
import java.util.Objects;
1819
import java.util.regex.Pattern;
1920
import lombok.experimental.UtilityClass;
21+
import org.apache.calcite.rex.RexNode;
22+
import org.apache.calcite.sql.type.SqlTypeName;
23+
import org.opensearch.sql.calcite.CalcitePlanContext;
24+
import org.opensearch.sql.calcite.type.ExprSqlType;
25+
import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
2026
import org.opensearch.sql.data.model.ExprTimeValue;
2127
import org.opensearch.sql.data.model.ExprValue;
2228
import org.opensearch.sql.expression.function.FunctionProperties;
29+
import org.opensearch.sql.expression.function.PPLBuiltinOperators;
30+
31+
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_DATE;
32+
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIME;
33+
import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP;
2334

2435
@UtilityClass
2536
public class DateTimeUtils {
@@ -294,4 +305,55 @@ private static String normalizeUnit(String rawUnit) {
294305
}
295306
}
296307
}
308+
309+
public static RexNode transferCompareForDateRelated(
310+
RexNode candidate, CalcitePlanContext context, SqlTypeName castTarget) {
311+
if (!(Objects.isNull(castTarget))) {
312+
switch (castTarget) {
313+
case DATE:
314+
if (!(candidate.getType() instanceof ExprSqlType
315+
&& ((ExprSqlType) candidate.getType()).getUdt() == EXPR_DATE)) {
316+
return context.rexBuilder.makeCall(PPLBuiltinOperators.DATE, candidate);
317+
}
318+
break;
319+
case TIME:
320+
if (!(candidate.getType() instanceof ExprSqlType
321+
&& ((ExprSqlType) candidate.getType()).getUdt() == EXPR_TIME)) {
322+
return context.rexBuilder.makeCall(PPLBuiltinOperators.TIME, candidate);
323+
}
324+
break;
325+
case TIMESTAMP:
326+
if (!(candidate.getType() instanceof ExprSqlType
327+
&& ((ExprSqlType) candidate.getType()).getUdt() == EXPR_TIMESTAMP)) {
328+
return context.rexBuilder.makeCall(PPLBuiltinOperators.TIMESTAMP, candidate);
329+
}
330+
break;
331+
default:
332+
return candidate;
333+
}
334+
}
335+
return candidate;
336+
}
337+
338+
public static SqlTypeName findCastType(RexNode left, RexNode right) {
339+
SqlTypeName leftType = returnCorrespondingSqlType(left);
340+
SqlTypeName rightType = returnCorrespondingSqlType(right);
341+
if (leftType != null && rightType != null) {
342+
return SqlTypeName.TIMESTAMP;
343+
}
344+
return leftType == null ? rightType : leftType;
345+
}
346+
347+
public static SqlTypeName returnCorrespondingSqlType(RexNode node) {
348+
if (node.getType() instanceof ExprSqlType) {
349+
OpenSearchTypeFactory.ExprUDT udt = ((ExprSqlType) node.getType()).getUdt();
350+
return switch (udt) {
351+
case EXPR_DATE -> SqlTypeName.DATE;
352+
case EXPR_TIME -> SqlTypeName.TIME;
353+
case EXPR_TIMESTAMP -> SqlTypeName.TIMESTAMP;
354+
default -> null;
355+
};
356+
}
357+
return null;
358+
}
297359
}

‎integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public void init() throws Exception {
2020
super.init();
2121
loadIndex(Index.ACCOUNT);
2222
loadIndex(Index.BANK);
23+
loadIndex(Index.DATE_FORMATS);
2324
}
2425

2526
@Test
@@ -74,6 +75,38 @@ public void testFilterByCompareStringTimestampPushDownExplain() throws IOExcepti
7475
+ "| where birthdate < '2018-11-09 00:00:00.000000000' "));
7576
}
7677

78+
@Test
79+
public void testFilterByCompareStringDatePushDownExplain() throws IOException {
80+
String expected =
81+
isCalciteEnabled()
82+
? loadFromFile(
83+
"expectedOutput/calcite/explain_filter_push_compare_date_string.json")
84+
: loadFromFile("expectedOutput/ppl/explain_filter_push_compare_date_string.json");
85+
86+
assertJsonEqualsIgnoreId(
87+
expected,
88+
explainQueryToString(
89+
"source=opensearch-sql_test_index_date_formats | fields yyyy-MM-dd"
90+
+ "| where yyyy-MM-dd > '2016-12-08 00:00:00.123456789' "
91+
+ "| where yyyy-MM-dd < '2018-11-09 00:00:00.000000000' "));
92+
}
93+
94+
@Test
95+
public void testFilterByCompareStringTimePushDownExplain() throws IOException {
96+
String expected =
97+
isCalciteEnabled()
98+
? loadFromFile(
99+
"expectedOutput/calcite/explain_filter_push_compare_time_string.json")
100+
: loadFromFile("expectedOutput/ppl/explain_filter_push_compare_time_string.json");
101+
102+
assertJsonEqualsIgnoreId(
103+
expected,
104+
explainQueryToString(
105+
"source=opensearch-sql_test_index_date_formats | fields custom_time"
106+
+ "| where custom_time > '2016-12-08 12:00:00.123456789' "
107+
+ "| where custom_time < '2018-11-09 19:00:00.123456789' "));
108+
}
109+
77110
@Test
78111
public void testFilterAndAggPushDownExplain() throws IOException {
79112
String expected =

‎integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite" : {
3+
"logical" : "LogicalFilter(condition=[<($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR))])\n LogicalFilter(condition=[>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR))])\n LogicalProject(yyyy-MM-dd=[$83])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n",
4+
"physical" : "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[yyyy-MM-dd], FILTER->>($0, '2016-12-08'), FILTER-><($0, '2018-11-09')], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"yyyy-MM-dd\":{\"from\":\"2016-12-08\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"yyyy-MM-dd\":{\"from\":null,\"to\":\"2018-11-09\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite" : {
3+
"logical" : "LogicalFilter(condition=[<($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR))])\n LogicalFilter(condition=[>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR))])\n LogicalProject(custom_time=[$49])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n",
4+
"physical" : "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[custom_time], FILTER->>($0, '12:00:00.123456789'), FILTER-><($0, '19:00:00.123456789')], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"custom_time\":{\"from\":\"12:00:00.123456789\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"custom_time\":{\"from\":null,\"to\":\"19:00:00.123456789\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"custom_time\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"root" : {
3+
"name" : "ProjectOperator",
4+
"description" : {
5+
"fields" : "[yyyy-MM-dd]"
6+
},
7+
"children" : [ {
8+
"name" : "FilterOperator",
9+
"description" : {
10+
"conditions" : "and(<(yyyy-MM-dd, cast_to_date(\"2018-11-09 00:00:00.000000000\")), >(yyyy-MM-dd, cast_to_date(\"2016-12-08 00:00:00.123456789\")))"
11+
},
12+
"children" : [ {
13+
"name" : "ProjectOperator",
14+
"description" : {
15+
"fields" : "[yyyy-MM-dd]"
16+
},
17+
"children" : [ {
18+
"name" : "OpenSearchIndexScan",
19+
"description" : {
20+
"request" : "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=s9y3QQEmb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9kYXRlX2Zvcm1hdHMWNXlYSjJyR1VScENzVW9JcWpHMS12ZwAWak9VTVBiNnNRemFNR015bERWWWdRUQAAAAAAAAAAARZtQUN3TWs2d1FXbUZsbnNxYl9KbHNnARY1eVhKMnJHVVJwQ3NVb0lxakcxLXZnAAA=, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
21+
},
22+
"children" : [ ]
23+
} ]
24+
} ]
25+
} ]
26+
}
27+
}

‎integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.json‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"root" : {
3+
"name" : "ProjectOperator",
4+
"description" : {
5+
"fields" : "[custom_time]"
6+
},
7+
"children" : [ {
8+
"name" : "FilterOperator",
9+
"description" : {
10+
"conditions" : "and(<(custom_time, cast_to_time(\"2018-11-09 19:00:00.123456789\")), >(custom_time, cast_to_time(\"2016-12-08 12:00:00.123456789\")))"
11+
},
12+
"children" : [ {
13+
"name" : "ProjectOperator",
14+
"description" : {
15+
"fields" : "[custom_time]"
16+
},
17+
"children" : [ {
18+
"name" : "OpenSearchIndexScan",
19+
"description" : {
20+
"request" : "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"custom_time\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=s9y3QQEmb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9kYXRlX2Zvcm1hdHMWM2VGc1RiM2VRV0NYeFJyRGpMTHI1QQAWdXpkcXIzSWZSbU9xZDdrNzk2b3JkdwAAAAAAAAAAARZJUno2SEQ1RVN6Uy1JeGN4RE9HeUtBARYzZUZzVGIzZVFXQ1h4UnJEakxMcjVBAAA=, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
21+
},
22+
"children" : [ ]
23+
} ]
24+
} ]
25+
} ]
26+
}
27+
}

0 commit comments

Comments
 (0)