Skip to content

Commit 2cdc165

Browse files
LantaoJinLantaoJin
committed
refactor
Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent 6da727a commit 2cdc165

98 files changed

Lines changed: 409 additions & 355 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/user/optimization/optimization.rst‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ The consecutive Filter operator will be merged as one Filter operator::
4444
{
4545
"name": "OpenSearchIndexScan",
4646
"description": {
47-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
47+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}})"
4848
},
4949
"children": []
5050
}
@@ -71,7 +71,7 @@ The Filter operator should be push down under Sort operator::
7171
{
7272
"name": "OpenSearchIndexScan",
7373
"description": {
74-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
74+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]})"
7575
},
7676
"children": []
7777
}
@@ -102,7 +102,7 @@ The Project list will push down to Query DSL to `filter the source <https://www.
102102
{
103103
"name": "OpenSearchIndexScan",
104104
"description": {
105-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
105+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}})"
106106
},
107107
"children": []
108108
}
@@ -128,7 +128,7 @@ The Filter operator will merge into OpenSearch Query DSL::
128128
{
129129
"name": "OpenSearchIndexScan",
130130
"description": {
131-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
131+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}})"
132132
},
133133
"children": []
134134
}
@@ -154,7 +154,7 @@ The Sort operator will merge into OpenSearch Query DSL::
154154
{
155155
"name": "OpenSearchIndexScan",
156156
"description": {
157-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)"
157+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]})"
158158
},
159159
"children": []
160160
}
@@ -188,7 +188,7 @@ Because the OpenSearch Script Based Sorting can't handle NULL/MISSING value, the
188188
{
189189
"name": "OpenSearchIndexScan",
190190
"description": {
191-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)"
191+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"})"
192192
},
193193
"children": []
194194
}
@@ -216,7 +216,7 @@ The Limit operator will merge in OpenSearch Query DSL::
216216
{
217217
"name": "OpenSearchIndexScan",
218218
"description": {
219-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":5,\"size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)"
219+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":5,\"size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}})"
220220
},
221221
"children": []
222222
}
@@ -252,7 +252,7 @@ If sort that includes expression, which cannot be merged into query DSL, also ex
252252
{
253253
"name": "OpenSearchIndexScan",
254254
"description": {
255-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, searchDone=false)"
255+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"})"
256256
},
257257
"children": []
258258
}
@@ -280,7 +280,7 @@ The Aggregation operator will merge into OpenSearch Aggregation::
280280
{
281281
"name": "OpenSearchIndexScan",
282282
"description": {
283-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
283+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}})"
284284
},
285285
"children": []
286286
}
@@ -306,7 +306,7 @@ The Sort operator will merge into OpenSearch Aggregation.::
306306
{
307307
"name": "OpenSearchIndexScan",
308308
"description": {
309-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
309+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}})"
310310
},
311311
"children": []
312312
}
@@ -341,7 +341,7 @@ Because the OpenSearch Composite Aggregation doesn't support order by metrics fi
341341
{
342342
"name": "OpenSearchIndexScan",
343343
"description": {
344-
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}}, searchDone=false)"
344+
"request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(age)\":{\"avg\":{\"field\":\"age\"}}}}}})"
345345
},
346346
"children": []
347347
}

‎docs/user/ppl/cmd/explain.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ Explain::
4848
{
4949
"name": "OpenSearchIndexScan",
5050
"description": {
51-
"request": """OpenSearchQueryRequest(indexName=state_country, sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"term":{"country":{"value":"USA","boost":1.0}}},{"term":{"country":{"value":"England","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"country":{"terms":{"field":"country","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"""
51+
"request": """OpenSearchQueryRequest(indexName=state_country, sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"should":[{"term":{"country":{"value":"USA","boost":1.0}}},{"term":{"country":{"value":"England","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"country":{"terms":{"field":"country","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"""
5252
},
5353
"children": []
5454
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchPaginatingIT.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.tpch;
7+
8+
import org.junit.After;
9+
import org.opensearch.sql.util.Retry;
10+
11+
@Retry
12+
public class CalcitePPLTpchPaginatingIT extends CalcitePPLTpchIT {
13+
14+
@Override
15+
public void init() throws Exception {
16+
super.init();
17+
setQueryBucketSize(2);
18+
}
19+
20+
@After
21+
public void tearDown() throws Exception {
22+
resetQueryBucketSize();
23+
super.tearDown();
24+
}
25+
}

‎integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ public static void assertJsonEqualsIgnoreId(String expected, String actual) {
421421
}
422422

423423
private static String cleanUpId(String s) {
424-
return eliminateTimeStamp(eliminatePid(eliminateRelId(s)));
424+
return eliminateTimeStamp(eliminatePid(eliminateRelId(eliminateRequestOptions(s))));
425425
}
426426

427427
private static String eliminateTimeStamp(String s) {
@@ -434,14 +434,17 @@ private static String eliminateRelId(String s) {
434434
.replaceAll("LogicalProject#\\d+", "LogicalProject#");
435435
}
436436

437+
private static String eliminateRequestOptions(String s) {
438+
return s.replaceAll(" needClean=true,", "").replaceAll(" searchDone=false,", "");
439+
}
440+
437441
private static String eliminatePid(String s) {
438442
return s.replaceAll("pitId=[^,]+,", "pitId=*,");
439443
}
440444

441445
/** Compare two YAML strings are equals with ignoring the RelNode id in the Calcite plan. */
442446
public static void assertYamlEqualsIgnoreId(String expectedYaml, String actualYaml) {
443-
String cleanedYaml = cleanUpYaml(actualYaml);
444-
assertYamlEquals(expectedYaml, cleanedYaml);
447+
assertYamlEquals(cleanUpYaml(expectedYaml), cleanUpYaml(actualYaml));
445448
}
446449

447450
public static void assertYamlEquals(String expected, String actual) {
@@ -460,7 +463,9 @@ private static String cleanUpYaml(String s) {
460463
.replaceAll("rel#\\d+", "rel#")
461464
.replaceAll("RelSubset#\\d+", "RelSubset#")
462465
.replaceAll("LogicalProject#\\d+", "LogicalProject#")
463-
.replaceAll("pitId=[^,]+,", "pitId=*,");
466+
.replaceAll("pitId=[^,]+,", "pitId=*,")
467+
.replaceAll(" needClean=true,", "")
468+
.replaceAll(" searchDone=false,", "");
464469
}
465470

466471
private static String jsonToYaml(String json) {

‎integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_from.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
calcite:
2+
logical: |
3+
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
4+
LogicalSort(offset=[2], fetch=[10])
5+
LogicalProject(c=[$1], state=[$0])
6+
LogicalAggregate(group=[{0}], c=[COUNT()])
7+
LogicalProject(state=[$7])
8+
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])
9+
physical: |
10+
EnumerableLimit(fetch=[10000])
11+
EnumerableLimit(offset=[2], fetch=[10])
12+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])

‎integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,5 @@ root:
1111
,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\
1212
,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\
1313
:[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\
14-
}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\
15-
\ searchAfter=null, searchResponse=null)"
14+
}}]}, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
1615
children: []

‎integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_can_match_shortcut.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@ root:
1515
:[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\
1616
,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\
1717
aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\
18-
asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\
19-
\ cursorKeepAlive=null, searchAfter=null, searchResponse=null)"
18+
asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=1m, searchAfter=null,\
19+
\ searchResponse=null)"
2020
children: []

‎integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_no_can_match_shortcut.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,6 @@ root:
1515
:[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\
1616
,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\
1717
aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\
18-
asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\
19-
\ cursorKeepAlive=null, searchAfter=null, searchResponse=null)"
18+
asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=1m, searchAfter=null,\
19+
\ searchResponse=null)"
2020
children: []

‎integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_with_after_timestamp.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,5 @@ root:
1111
,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\
1212
,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\
1313
:[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\
14-
}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\
15-
\ searchAfter=null, searchResponse=null)"
14+
}}]}, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
1615
children: []

‎integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ root:
77
description:
88
request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\
99
size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`agent.name`)\":{\"cardinality\"\
10-
:{\"field\":\"agent.name\"}}}}, needClean=true, searchDone=false, pitId=*,\
11-
\ cursorKeepAlive=null, searchAfter=null, searchResponse=null)"
10+
:{\"field\":\"agent.name\"}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\
11+
\ searchResponse=null)"
1212
children: []

0 commit comments

Comments
 (0)