Skip to content

Commit 0341c89

Browse files
yuancuyuancu
committed
Support serializing & deserializing UDTs when pushing down scripts (opensearch-project#4245)
* Support serializing & deserializing UDTs Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Update explain ITs Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Push down UDT types as string types for comparison operators Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Separate test cases and add an ignored IT Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Correct the handling of UDT in CalciteScriptEngine by substituting calcite's type factory with OpenSearchTypeFactory Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Fix deserialization for IP Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Remove testExplainPushDownScriptsContainingUDT in v2 Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Enable testLimitAfterAggregation in CalcitePPLAggregationIT Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Unit test serialize map and array types Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Fix deeper level deserialization of UDTs Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Add a yaml test for issue 4322 Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Add a test case for issue 4340 Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> * Remove redundant classes Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> --------- Signed-off-by: Yuanchun Shen <yuanchu@amazon.com> (cherry picked from commit 69a718b)
1 parent 2c41580 commit 0341c89

16 files changed

Lines changed: 1521 additions & 14 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,6 @@ public void testBinOnlyWithoutAggregation() throws IOException {
204204
}
205205

206206
@Test
207-
@Ignore
208-
// https://github.com/opensearch-project/sql/issues/4063
209207
public void testBinWithTimestampAggregation() throws IOException {
210208
// Test bin operation with fields only - no aggregation
211209
JSONObject result =

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS;
1111
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE;
1212
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS;
13+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS;
1314
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORKER;
1415
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORK_INFORMATION;
1516
import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId;
@@ -1082,6 +1083,33 @@ public void testStrftimeFunctionExplain() throws IOException {
10821083
assertJsonEqualsIgnoreId(expected, result);
10831084
}
10841085

1086+
// Script generation is not stable in v2
1087+
@Test
1088+
public void testExplainPushDownScriptsContainingUDT() throws IOException {
1089+
assertJsonEqualsIgnoreId(
1090+
loadExpectedPlan("explain_filter_script_ip_push.json"),
1091+
explainQueryToString(
1092+
String.format(
1093+
"source=%s | where cidrmatch(host, '0.0.0.0/24') | fields host",
1094+
TEST_INDEX_WEBLOGS)));
1095+
1096+
assertYamlEqualsJsonIgnoreId(
1097+
loadExpectedPlan("explain_agg_script_timestamp_push.yaml"),
1098+
explainQueryToString(
1099+
String.format(
1100+
"source=%s | eval t = unix_timestamp(birthdate) | stats count() by t | sort t |"
1101+
+ " head 3",
1102+
TEST_INDEX_BANK)));
1103+
1104+
assertJsonEqualsIgnoreId(
1105+
loadExpectedPlan("explain_agg_script_udt_arg_push.json"),
1106+
explainQueryToString(
1107+
String.format(
1108+
"source=%s | eval t = date_add(birthdate, interval 1 day) | stats count() by"
1109+
+ " span(t, 1d)",
1110+
TEST_INDEX_BANK)));
1111+
}
1112+
10851113
@Test
10861114
public void testFillNullValueSyntaxExplain() throws IOException {
10871115
String expected = loadExpectedPlan("explain_fillnull_value_syntax.yaml");

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1220,6 +1220,45 @@ public void testStatsMinOnStringField() throws IOException {
12201220
verifyDataRows(actual, rows("Amber JOHnny"));
12211221
}
12221222

1223+
@Test
1224+
public void testStatsCountOnFunctionsWithUDTArg() throws IOException {
1225+
JSONObject response =
1226+
executeQuery(
1227+
String.format(
1228+
"source=%s | eval t = unix_timestamp(birthdate) | stats count() by t | sort -t",
1229+
TEST_INDEX_BANK));
1230+
verifySchema(response, schema("count()", "bigint"), schema("t", "double"));
1231+
verifyDataRows(
1232+
response,
1233+
rows(1, 1542152000),
1234+
rows(1, 1534636800),
1235+
rows(1, 1533945600),
1236+
rows(1, 1530057600),
1237+
rows(1, 1529712000),
1238+
rows(1, 1511136000),
1239+
rows(1, 1508716800));
1240+
}
1241+
1242+
@Test
1243+
public void testStatsGroupByDate() throws IOException {
1244+
JSONObject resonse =
1245+
executeQuery(
1246+
String.format(
1247+
"source=%s | eval t = date_add(birthdate, interval 1 day) | stats count() by"
1248+
+ " span(t, 1d)",
1249+
TEST_INDEX_BANK));
1250+
verifySchema(resonse, schema("count()", "bigint"), schema("span(t,1d)", "timestamp"));
1251+
verifyDataRows(
1252+
resonse,
1253+
rows(1, "2017-10-24 00:00:00"),
1254+
rows(1, "2017-11-21 00:00:00"),
1255+
rows(1, "2018-06-24 00:00:00"),
1256+
rows(1, "2018-06-28 00:00:00"),
1257+
rows(1, "2018-08-12 00:00:00"),
1258+
rows(1, "2018-08-20 00:00:00"),
1259+
rows(1, "2018-11-14 00:00:00"));
1260+
}
1261+
12231262
@Test
12241263
public void testLimitAfterAggregation() throws IOException {
12251264
JSONObject response =

‎integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ calcite:
77
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])
88
physical: |
99
EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0])
10-
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBz3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAABN0AAhfcm91dGluZ35yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3QADmFjY291bnRfbnVtYmVyfnEAfgAKdAAETE9OR3QACWZpcnN0bmFtZXEAfgAMdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAp0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+AAx+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAGX2luZGV4cQB+AAx0AARjaXR5cQB+AAx0AAlfbWF4c2NvcmV+cQB+AAp0AAVGTE9BVHQABl9zY29yZXEAfgA4dAAFX3NvcnRxAH4AD3QACGxhc3RuYW1lcQB+AAx0AAdiYWxhbmNlcQB+AA90AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANfaWRxAH4ADHQAA2FnZX5xAH4ACnQAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4ACnQAB0JPT0xFQU54eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])
10+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAATdAAIX3JvdXRpbmd+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd0AA5hY2NvdW50X251bWJlcn5xAH4ACnQABExPTkd0AAlmaXJzdG5hbWVxAH4ADHQAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgAKdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAMfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABl9pbmRleHEAfgAMdAAEY2l0eXEAfgAMdAAJX21heHNjb3JlfnEAfgAKdAAFRkxPQVR0AAZfc2NvcmVxAH4AOHQABV9zb3J0cQB+AA90AAhsYXN0bmFtZXEAfgAMdAAHYmFsYW5jZXEAfgAPdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADX2lkcQB+AAx0AANhZ2V+cQB+AAp0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAp0AAdCT09MRUFOeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

‎integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$FIELD_INDEX])\n LogicalFilter(condition=[CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n",
4+
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfSVAiLAogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQB9XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIwLjAuMC4wLzI0IiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABGhvc3R+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAJJUHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(t,1d)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')])\n LogicalFilter(condition=[IS NOT NULL($19)])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n",
4+
"physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(t,1d)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:INTERVAL DAY], expr#20=[DATE_ADD($t3, $t19)], expr#21=[1], expr#22=['d'], expr#23=[SPAN($t20, $t21, $t22)], expr#24=[IS NOT NULL($t20)], span(t,1d)=[$t23], $condition=[$t24])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n"
5+
}
6+
}

0 commit comments

Comments
 (0)