Skip to content

Commit b060479

Browse files
LantaoJinLantaoJin
committed
Fix IT
Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent f66d5a3 commit b060479

6 files changed

Lines changed: 63 additions & 21 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,7 @@ public void testStatsByTimeSpan() throws IOException {
428428
}
429429

430430
@Test
431-
public void testDedupPushdownV3() throws IOException {
431+
public void testDedupPushdown() throws IOException {
432432
String expected = loadExpectedPlan("explain_dedup_push.json");
433433
assertJsonEqualsIgnoreId(
434434
expected,
@@ -437,6 +437,16 @@ public void testDedupPushdownV3() throws IOException {
437437
+ " | dedup 1 gender"));
438438
}
439439

440+
@Test
441+
public void testDedupKeepEmptyPushdown() throws IOException {
442+
String expected = loadExpectedPlan("explain_dedup_keepempty_push.json");
443+
assertJsonEqualsIgnoreId(
444+
expected,
445+
explainQueryToString(
446+
"source=opensearch-sql_test_index_account | fields account_number, gender, age"
447+
+ " | dedup gender KEEPEMPTY=true"));
448+
}
449+
440450
@Test
441451
public void testSingleFieldRelevanceQueryFunctionExplain() throws IOException {
442452
// This test is only applicable if pushdown is enabled

‎integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_push.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], gender=[$1], age=[$2])\n LogicalFilter(condition=[OR(IS NULL($1), <=($3, 1))])\n LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4 ORDER BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
4+
"physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[IS NULL($t1)], expr#5=[1], expr#6=[<=($t3, $t5)], expr#7=[OR($t4, $t6)], proj#0..2=[{exprs}], $condition=[$t7])\n EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"gender\",\"age\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"calcite": {
33
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], gender=[$1], age=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(account_number=[$0], gender=[$1], age=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $1 ORDER BY $1)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(account_number=[$0], gender=[$4], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
4-
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], COLLAPSE->gender, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"gender\",\"age\"],\"excludes\":[]},\"collapse\":{\"field\":\"gender.keyword\"}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
4+
"physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], FILTER->IS NOT NULL($1), COLLAPSE->gender, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"gender\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"collapse\":{\"field\":\"gender.keyword\"}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
55
}
66
}

‎integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_dedup_keepempty_push.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"calcite": {
3+
"logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], gender=[$1], age=[$2])\n LogicalFilter(condition=[OR(IS NULL($1), <=($3, 1))])\n LogicalProject(account_number=[$0], gender=[$4], age=[$8], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $4 ORDER BY $4)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
4+
"physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..17=[{inputs}], expr#18=[IS NULL($t4)], expr#19=[1], expr#20=[<=($t17, $t19)], expr#21=[OR($t18, $t20)], account_number=[$t0], gender=[$t4], age=[$t8], $condition=[$t21])\n EnumerableWindow(window#0=[window(partition {4} order by [4] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"
5+
}
6+
}

‎integ-test/src/test/resources/expectedOutput/ppl/explain_dedup_keepempty_push.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"root": {
3+
"name": "ProjectOperator",
4+
"description": {
5+
"fields": "[account_number, gender, age]"
6+
},
7+
"children": [{
8+
"name": "DedupeOperator",
9+
"description": {
10+
"dedupeList": "[gender]",
11+
"allowedDuplication": 1,
12+
"keepEmpty": true,
13+
"consecutive": false
14+
},
15+
"children": [{
16+
"name": "ProjectOperator",
17+
"description": {
18+
"fields": "[account_number, gender, age]"
19+
},
20+
"children": [{
21+
"name": "OpenSearchIndexScan",
22+
"description": {
23+
"request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"gender\",\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)"
24+
},
25+
"children": []
26+
}]
27+
}]
28+
}]
29+
}
30+
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,7 @@ public void onMatch(RelOptRuleCall call) {
3232
final LogicalProject finalOutput = call.rel(0);
3333
final LogicalFilter numOfDedupFilter = call.rel(1);
3434
final LogicalProject projectWithWindow = call.rel(2);
35-
final LogicalFilter isNotNullFilter = call.rel(3);
36-
final CalciteLogicalIndexScan scan = call.rel(4);
35+
final CalciteLogicalIndexScan scan = call.rel(3);
3736
RexLiteral numLiteral =
3837
PlanUtils.findLiterals(numOfDedupFilter.getCondition(), true).getFirst();
3938
Integer num = numLiteral.getValueAs(Integer.class);
@@ -90,24 +89,15 @@ public interface Config extends RelRule.Config {
9089
.predicate(PlanUtils::containsRowNumberDedup)
9190
.oneInput(
9291
b3 ->
93-
b3.operand(LogicalFilter.class)
92+
b3.operand(CalciteLogicalIndexScan.class)
9493
.predicate(
95-
f ->
96-
f.getCondition().getKind()
97-
== SqlKind.IS_NOT_NULL)
98-
.oneInput(
99-
b4 ->
100-
b4.operand(
101-
CalciteLogicalIndexScan
102-
.class)
103-
.predicate(
104-
Predicate.not(
105-
OpenSearchIndexScanRule
106-
::isLimitPushed)
107-
.and(
108-
OpenSearchIndexScanRule
109-
::noAggregatePushed))
110-
.noInputs())))));
94+
Predicate.not(
95+
OpenSearchIndexScanRule
96+
::isLimitPushed)
97+
.and(
98+
OpenSearchIndexScanRule
99+
::noAggregatePushed))
100+
.noInputs()))));
111101

112102
@Override
113103
default OpenSearchDedupPushdownRule toRule() {

0 commit comments

Comments
 (0)