Skip to content

Commit c12ed27

Browse files
opensearch-trigger-bot[bot]github-actions[bot]Selina Song
authored
Support reverse command with Calcite (opensearch-project#3867) (opensearch-project#3991)
* Implement reverse * Add reverse integ tests and unit tests * Add reverse documentation * Modify reverse test and documentation * Fix limit pushdown bug when reverse comes before head * Revert "Fix limit pushdown bug when reverse comes before head" This reverts commit 087c936. * Fix grammar, naming, and test cases. Pushdown reverted will be in 2nd PR. * Fix reverse tests: update logical plans, format with Spotless - Updated expected logical plans and Spark SQL in reverse tests - Applied Spotless to fix formatting * Fix OS version in build 3.1.0 * Add note on limitation to rst * Move explain IT to correct file, add Anonymizer test * Add reverse to index.rst --------- (cherry picked from commit c05a58c) Signed-off-by: Selina Song <selsong@amazon.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Selina Song <selsong@amazon.com>
1 parent 5ec26e6 commit c12ed27

13 files changed

Lines changed: 541 additions & 0 deletions

File tree

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@
8080
import org.opensearch.sql.ast.tree.Relation;
8181
import org.opensearch.sql.ast.tree.RelationSubquery;
8282
import org.opensearch.sql.ast.tree.Rename;
83+
import org.opensearch.sql.ast.tree.Reverse;
8384
import org.opensearch.sql.ast.tree.Sort;
8485
import org.opensearch.sql.ast.tree.Sort.SortOption;
8586
import org.opensearch.sql.ast.tree.SubqueryAlias;
@@ -681,6 +682,12 @@ public LogicalPlan visitFlatten(Flatten node, AnalysisContext context) {
681682
"FLATTEN is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
682683
}
683684

685+
@Override
686+
public LogicalPlan visitReverse(Reverse node, AnalysisContext context) {
687+
throw new UnsupportedOperationException(
688+
"REVERSE is supported only when " + CALCITE_ENGINE_ENABLED.getKeyValue() + "=true");
689+
}
690+
684691
@Override
685692
public LogicalPlan visitPaginate(Paginate paginate, AnalysisContext context) {
686693
LogicalPlan child = paginate.getChild().get(0).accept(this, context);

‎core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
import org.opensearch.sql.ast.tree.Relation;
7070
import org.opensearch.sql.ast.tree.RelationSubquery;
7171
import org.opensearch.sql.ast.tree.Rename;
72+
import org.opensearch.sql.ast.tree.Reverse;
7273
import org.opensearch.sql.ast.tree.Sort;
7374
import org.opensearch.sql.ast.tree.SubqueryAlias;
7475
import org.opensearch.sql.ast.tree.TableFunction;
@@ -244,6 +245,10 @@ public T visitSort(Sort node, C context) {
244245
return visitChildren(node, context);
245246
}
246247

248+
public T visitReverse(Reverse node, C context) {
249+
return visitChildren(node, context);
250+
}
251+
247252
public T visitLambdaFunction(LambdaFunction node, C context) {
248253
return visitChildren(node, context);
249254
}

‎core/src/main/java/org/opensearch/sql/ast/tree/Reverse.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ast.tree;
7+
8+
import com.google.common.collect.ImmutableList;
9+
import java.util.List;
10+
import lombok.EqualsAndHashCode;
11+
import lombok.Getter;
12+
import lombok.RequiredArgsConstructor;
13+
import lombok.Setter;
14+
import lombok.ToString;
15+
import org.opensearch.sql.ast.AbstractNodeVisitor;
16+
17+
/** AST node represent Reverse operation. */
18+
@Getter
19+
@Setter
20+
@ToString
21+
@EqualsAndHashCode(callSuper = false)
22+
@RequiredArgsConstructor
23+
public class Reverse extends UnresolvedPlan {
24+
25+
private UnresolvedPlan child;
26+
27+
@Override
28+
public Reverse attach(UnresolvedPlan child) {
29+
this.child = child;
30+
return this;
31+
}
32+
33+
@Override
34+
public List<UnresolvedPlan> getChild() {
35+
return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child);
36+
}
37+
38+
@Override
39+
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
40+
return nodeVisitor.visitReverse(this, context);
41+
}
42+
}

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,28 @@ public RelNode visitHead(Head node, CalcitePlanContext context) {
359359
return context.relBuilder.peek();
360360
}
361361

362+
private static final String REVERSE_ROW_NUM = "__reverse_row_num__";
363+
364+
@Override
365+
public RelNode visitReverse(
366+
org.opensearch.sql.ast.tree.Reverse node, CalcitePlanContext context) {
367+
visitChildren(node, context);
368+
// Add ROW_NUMBER() column
369+
RexNode rowNumber =
370+
context
371+
.relBuilder
372+
.aggregateCall(SqlStdOperatorTable.ROW_NUMBER)
373+
.over()
374+
.rowsTo(RexWindowBounds.CURRENT_ROW)
375+
.as(REVERSE_ROW_NUM);
376+
context.relBuilder.projectPlus(rowNumber);
377+
// Sort by row number descending
378+
context.relBuilder.sort(context.relBuilder.desc(context.relBuilder.field(REVERSE_ROW_NUM)));
379+
// Remove row number column
380+
context.relBuilder.projectExcept(context.relBuilder.field(REVERSE_ROW_NUM));
381+
return context.relBuilder.peek();
382+
}
383+
362384
@Override
363385
public RelNode visitParse(Parse node, CalcitePlanContext context) {
364386
visitChildren(node, context);

‎docs/user/ppl/cmd/reverse.rst‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
=============
2+
reverse
3+
=============
4+
5+
.. rubric:: Table of contents
6+
7+
.. contents::
8+
:local:
9+
:depth: 2
10+
11+
12+
Description
13+
============
14+
| Using ``reverse`` command to reverse the display order of search results. The same results are returned, but in reverse order.
15+
16+
Version
17+
=======
18+
3.2.0
19+
20+
Syntax
21+
============
22+
reverse
23+
24+
25+
* No parameters: The reverse command takes no arguments or options.
26+
27+
Note
28+
=====
29+
The `reverse` command processes the entire dataset. If applied directly to millions of records, it will consume significant memory resources on the coordinating node. Users should only apply the `reverse` command to smaller datasets, typically after aggregation operations.
30+
31+
Example 1: Basic reverse operation
32+
==================================
33+
34+
The example shows reversing the order of all documents.
35+
36+
PPL query::
37+
38+
os> source=accounts | fields account_number, age | reverse;
39+
fetched rows / total rows = 4/4
40+
+----------------+-----+
41+
| account_number | age |
42+
|----------------+-----|
43+
| 6 | 36 |
44+
| 18 | 33 |
45+
| 1 | 32 |
46+
| 13 | 28 |
47+
+----------------+-----+
48+
49+
50+
Example 2: Reverse with sort
51+
============================
52+
53+
The example shows reversing results after sorting by age in ascending order, effectively giving descending order.
54+
55+
PPL query::
56+
57+
os> source=accounts | sort age | fields account_number, age | reverse;
58+
fetched rows / total rows = 4/4
59+
+----------------+-----+
60+
| account_number | age |
61+
|----------------+-----|
62+
| 6 | 36 |
63+
| 18 | 33 |
64+
| 1 | 32 |
65+
| 13 | 28 |
66+
+----------------+-----+
67+
68+
69+
Example 3: Reverse with head
70+
============================
71+
72+
The example shows using reverse with head to get the last 2 records from the original order.
73+
74+
PPL query::
75+
76+
os> source=accounts | reverse | head 2 | fields account_number, age;
77+
fetched rows / total rows = 2/2
78+
+----------------+-----+
79+
| account_number | age |
80+
|----------------+-----|
81+
| 6 | 36 |
82+
| 18 | 33 |
83+
+----------------+-----+
84+
85+
86+
Example 4: Double reverse
87+
=========================
88+
89+
The example shows that applying reverse twice returns to the original order.
90+
91+
PPL query::
92+
93+
os> source=accounts | reverse | reverse | fields account_number, age;
94+
fetched rows / total rows = 4/4
95+
+----------------+-----+
96+
| account_number | age |
97+
|----------------+-----|
98+
| 13 | 28 |
99+
| 1 | 32 |
100+
| 18 | 33 |
101+
| 6 | 36 |
102+
+----------------+-----+
103+
104+
105+
Example 5: Reverse with complex pipeline
106+
=======================================
107+
108+
The example shows reverse working with filtering and field selection.
109+
110+
PPL query::
111+
112+
os> source=accounts | where age > 30 | fields account_number, age | reverse;
113+
fetched rows / total rows = 3/3
114+
+----------------+-----+
115+
| account_number | age |
116+
|----------------+-----|
117+
| 6 | 36 |
118+
| 18 | 33 |
119+
| 1 | 32 |
120+
+----------------+-----+

‎docs/user/ppl/index.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ The query start with search command and then flowing a set of command delimited
104104

105105
- `subquery (aka subsearch) command <cmd/subquery.rst>`_
106106

107+
- `reverse command <cmd/reverse.rst>`_
108+
107109
- `top command <cmd/top.rst>`_
108110

109111
- `trendline command <cmd/trendline.rst>`_

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,4 +119,31 @@ public void testFilterScriptPushDownExplain() throws Exception {
119119
public void testFilterFunctionScriptPushDownExplain() throws Exception {
120120
super.testFilterFunctionScriptPushDownExplain();
121121
}
122+
123+
@Test
124+
public void testExplainWithReverse() throws IOException {
125+
String result =
126+
executeWithReplace(
127+
"explain source=opensearch-sql_test_index_account | sort age | reverse | head 5");
128+
129+
// Verify that the plan contains a LogicalSort with fetch (from head 5)
130+
assertTrue(result.contains("LogicalSort") && result.contains("fetch=[5]"));
131+
132+
// Verify that reverse added a ROW_NUMBER and another sort (descending)
133+
assertTrue(result.contains("ROW_NUMBER()"));
134+
assertTrue(result.contains("dir0=[DESC]"));
135+
}
136+
137+
/**
138+
* Executes the PPL query and returns the result as a string with windows-style line breaks
139+
* replaced with Unix-style ones.
140+
*
141+
* @param ppl the PPL query to execute
142+
* @return the result of the query as a string with line breaks replaced
143+
* @throws IOException if an error occurs during query execution
144+
*/
145+
private String executeWithReplace(String ppl) throws IOException {
146+
var result = executeQueryToString(ppl);
147+
return result.replace("\\r\\n", "\\n");
148+
}
122149
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteReverseCommandIT.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.remote;
7+
8+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
9+
import static org.opensearch.sql.util.MatcherUtils.rows;
10+
import static org.opensearch.sql.util.MatcherUtils.schema;
11+
import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder;
12+
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
13+
14+
import java.io.IOException;
15+
import org.json.JSONObject;
16+
import org.junit.jupiter.api.Test;
17+
import org.opensearch.sql.ppl.PPLIntegTestCase;
18+
19+
public class CalciteReverseCommandIT extends PPLIntegTestCase {
20+
21+
@Override
22+
public void init() throws Exception {
23+
super.init();
24+
enableCalcite();
25+
disallowCalciteFallback();
26+
loadIndex(Index.BANK);
27+
}
28+
29+
@Test
30+
public void testReverse() throws IOException {
31+
JSONObject result =
32+
executeQuery(String.format("source=%s | fields account_number | reverse", TEST_INDEX_BANK));
33+
verifySchema(result, schema("account_number", "bigint"));
34+
verifyDataRowsInOrder(
35+
result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1));
36+
}
37+
38+
@Test
39+
public void testReverseWithFields() throws IOException {
40+
JSONObject result =
41+
executeQuery(
42+
String.format(
43+
"source=%s | fields account_number, firstname | reverse", TEST_INDEX_BANK));
44+
verifySchema(result, schema("account_number", "bigint"), schema("firstname", "string"));
45+
verifyDataRowsInOrder(
46+
result,
47+
rows(32, "Dillard"),
48+
rows(25, "Virginia"),
49+
rows(20, "Elinor"),
50+
rows(18, "Dale"),
51+
rows(13, "Nanette"),
52+
rows(6, "Hattie"),
53+
rows(1, "Amber JOHnny"));
54+
}
55+
56+
@Test
57+
public void testReverseWithSort() throws IOException {
58+
JSONObject result =
59+
executeQuery(
60+
String.format(
61+
"source=%s | sort account_number | fields account_number | reverse",
62+
TEST_INDEX_BANK));
63+
verifySchema(result, schema("account_number", "bigint"));
64+
verifyDataRowsInOrder(
65+
result, rows(32), rows(25), rows(20), rows(18), rows(13), rows(6), rows(1));
66+
}
67+
68+
@Test
69+
public void testDoubleReverse() throws IOException {
70+
JSONObject result =
71+
executeQuery(
72+
String.format(
73+
"source=%s | fields account_number | reverse | reverse", TEST_INDEX_BANK));
74+
verifySchema(result, schema("account_number", "bigint"));
75+
verifyDataRowsInOrder(
76+
result, rows(1), rows(6), rows(13), rows(18), rows(20), rows(25), rows(32));
77+
}
78+
79+
@Test
80+
public void testReverseWithHead() throws IOException {
81+
JSONObject result =
82+
executeQuery(
83+
String.format("source=%s | fields account_number | reverse | head 3", TEST_INDEX_BANK));
84+
verifySchema(result, schema("account_number", "bigint"));
85+
verifyDataRowsInOrder(result, rows(32), rows(25), rows(20));
86+
}
87+
88+
@Test
89+
public void testReverseWithComplexPipeline() throws IOException {
90+
JSONObject result =
91+
executeQuery(
92+
String.format(
93+
"source=%s | where account_number > 18 | fields account_number | reverse | head 2",
94+
TEST_INDEX_BANK));
95+
verifySchema(result, schema("account_number", "bigint"));
96+
verifyDataRowsInOrder(result, rows(32), rows(25));
97+
}
98+
99+
@Test
100+
public void testReverseWithMultipleSorts() throws IOException {
101+
// Use the existing BANK data but with a simpler, more predictable query
102+
JSONObject result =
103+
executeQuery(
104+
String.format(
105+
"source=%s | sort account_number | fields account_number | reverse | head 3",
106+
TEST_INDEX_BANK));
107+
verifySchema(result, schema("account_number", "bigint"));
108+
verifyDataRowsInOrder(result, rows(32), rows(25), rows(20));
109+
}
110+
}

0 commit comments

Comments
 (0)