Skip to content

Commit 4587b5d

Browse files
yuancuyuancu
committed
Remove unused logical expand & add test cases
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent 10f81b0 commit 4587b5d

7 files changed

Lines changed: 239 additions & 40 deletions

File tree

‎core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,30 @@
4949
import org.opensearch.sql.ast.expression.When;
5050
import org.opensearch.sql.ast.expression.WindowFunction;
5151
import org.opensearch.sql.ast.expression.Xor;
52-
import org.opensearch.sql.ast.tree.*;
52+
import org.opensearch.sql.ast.tree.Aggregation;
53+
import org.opensearch.sql.ast.tree.Dedupe;
54+
import org.opensearch.sql.ast.tree.DescribeRelation;
55+
import org.opensearch.sql.ast.tree.Eval;
56+
import org.opensearch.sql.ast.tree.Expand;
57+
import org.opensearch.sql.ast.tree.FillNull;
58+
import org.opensearch.sql.ast.tree.Filter;
59+
import org.opensearch.sql.ast.tree.Head;
60+
import org.opensearch.sql.ast.tree.Limit;
61+
import org.opensearch.sql.ast.tree.Parse;
62+
import org.opensearch.sql.ast.tree.Patterns;
63+
import org.opensearch.sql.ast.tree.Project;
64+
import org.opensearch.sql.ast.tree.RareTopN;
5365
import org.opensearch.sql.ast.tree.RareTopN.CommandType;
66+
import org.opensearch.sql.ast.tree.Relation;
67+
import org.opensearch.sql.ast.tree.RelationSubquery;
68+
import org.opensearch.sql.ast.tree.Rename;
69+
import org.opensearch.sql.ast.tree.Sort;
5470
import org.opensearch.sql.ast.tree.Sort.SortOption;
71+
import org.opensearch.sql.ast.tree.SubqueryAlias;
72+
import org.opensearch.sql.ast.tree.TableFunction;
73+
import org.opensearch.sql.ast.tree.Trendline;
74+
import org.opensearch.sql.ast.tree.UnresolvedPlan;
75+
import org.opensearch.sql.ast.tree.Values;
5576

5677
/** Class of static methods to create specific node instances. */
5778
@UtilityClass

‎core/src/main/java/org/opensearch/sql/ast/tree/Expand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,17 @@
88
import com.google.common.collect.ImmutableList;
99
import java.util.List;
1010
import javax.annotation.Nullable;
11+
import lombok.EqualsAndHashCode;
1112
import lombok.Getter;
1213
import lombok.RequiredArgsConstructor;
1314
import lombok.ToString;
1415
import org.opensearch.sql.ast.AbstractNodeVisitor;
1516
import org.opensearch.sql.ast.expression.Field;
1617

1718
/** AST node representing an {@code expand <field>} operation. */
18-
@Getter
1919
@ToString
2020
@RequiredArgsConstructor
21+
@EqualsAndHashCode(callSuper = false)
2122
public class Expand extends UnresolvedPlan {
2223

2324
private UnresolvedPlan child;

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1032,7 +1032,7 @@ public RelNode visitExpand(Expand expand, CalcitePlanContext context) {
10321032
// be used by the right side to correlate with the left side.
10331033
relBuilder.correlate(JoinRelType.INNER, correlVariable.get().id, List.of(arrayFieldRex));
10341034

1035-
// 8. Remove the original array field from the output.
1035+
// 9. Remove the original array field from the output.
10361036
// TODO: RFC: should we keep the original array field when alias is present?
10371037
relBuilder.projectExcept(arrayFieldRex);
10381038
if (alias != null) {

‎core/src/main/java/org/opensearch/sql/planner/logical/LogicalExpand.java‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,6 @@ public R visitAD(LogicalAD plan, C context) {
104104
return visitNode(plan, context);
105105
}
106106

107-
public R visitExpand(LogicalExpand plan, C context) {
108-
return visitNode(plan, context);
109-
}
110-
111107
public R visitTrendline(LogicalTrendline plan, C context) {
112108
return visitNode(plan, context);
113109
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java‎

Lines changed: 213 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,13 @@
99

1010
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ARRAY;
1111
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE;
12+
import static org.opensearch.sql.util.MatcherUtils.rows;
1213
import static org.opensearch.sql.util.MatcherUtils.schema;
14+
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
1315
import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows;
1416
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
1517

18+
import org.json.JSONArray;
1619
import org.json.JSONObject;
1720
import org.junit.Ignore;
1821
import org.junit.jupiter.api.Test;
@@ -39,11 +42,108 @@ public void testExpandOnNested() throws Exception {
3942
schema("id", "bigint"),
4043
schema("address", "struct"));
4144
verifyNumOfRows(response, 11);
45+
verifyDataRows(
46+
response,
47+
rows(
48+
"abbas",
49+
null,
50+
24,
51+
new JSONObject()
52+
.put("city", "New york city")
53+
.put("state", "NY")
54+
.put("moveInDate", new JSONObject().put("dateAndTime", "1984-04-12 09:07:42"))),
55+
rows(
56+
"abbas",
57+
null,
58+
24,
59+
new JSONObject()
60+
.put("city", "bellevue")
61+
.put("state", "WA")
62+
.put(
63+
"moveInDate",
64+
new JSONArray()
65+
.put(new JSONObject().put("dateAndTime", "2023-05-03 08:07:42"))
66+
.put(new JSONObject().put("dateAndTime", "2001-11-11 04:07:44")))),
67+
rows(
68+
"abbas",
69+
null,
70+
24,
71+
new JSONObject()
72+
.put("city", "seattle")
73+
.put("state", "WA")
74+
.put("moveInDate", new JSONObject().put("dateAndTime", "1966-03-19 03:04:55"))),
75+
rows(
76+
"abbas",
77+
null,
78+
24,
79+
new JSONObject()
80+
.put("city", "chicago")
81+
.put("state", "IL")
82+
.put("moveInDate", new JSONObject().put("dateAndTime", "2011-06-01 01:01:42"))),
83+
rows(
84+
"chen",
85+
null,
86+
32,
87+
new JSONObject()
88+
.put("city", "Miami")
89+
.put("state", "Florida")
90+
.put("moveInDate", new JSONObject().put("dateAndTime", "1901-08-11 04:03:33"))),
91+
rows(
92+
"chen",
93+
null,
94+
32,
95+
new JSONObject()
96+
.put("city", "los angeles")
97+
.put("state", "CA")
98+
.put("moveInDate", new JSONObject().put("dateAndTime", "2023-05-03 08:07:42"))),
99+
rows(
100+
"peng",
101+
null,
102+
26,
103+
new JSONObject()
104+
.put("city", "san diego")
105+
.put("state", "CA")
106+
.put("moveInDate", new JSONObject().put("dateAndTime", "2001-11-11 04:07:44"))),
107+
rows(
108+
"peng",
109+
null,
110+
26,
111+
new JSONObject()
112+
.put("city", "austin")
113+
.put("state", "TX")
114+
.put("moveInDate", new JSONObject().put("dateAndTime", "1977-07-13 09:04:41"))),
115+
rows(
116+
"andy",
117+
4,
118+
19,
119+
new JSONObject()
120+
.put("city", "houston")
121+
.put("state", "TX")
122+
.put("moveInDate", new JSONObject().put("dateAndTime", "1933-12-12 05:05:45"))),
123+
rows(
124+
"david",
125+
null,
126+
25,
127+
new JSONObject()
128+
.put("city", "raleigh")
129+
.put("state", "NC")
130+
.put("moveInDate", new JSONObject().put("dateAndTime", "1909-06-17 01:04:21"))),
131+
rows(
132+
"david",
133+
null,
134+
25,
135+
new JSONObject()
136+
.put("city", "charlotte")
137+
.put("state", "SC")
138+
.put(
139+
"moveInDate",
140+
new JSONArray()
141+
.put(new JSONObject().put("dateAndTime", "2001-11-11 04:07:44")))));
42142
}
43143

44-
// TODO: confirm if expand on array (instead of nested) will be supported.
144+
// To consider in future releases: will expand on array (instead of nested) be supported.
45145
// In Opensearch, a string field can store either a single string or an array of strings.
46-
// This makes it difficult to implement expand on array.
146+
// This makes it difficult to implement expand on arries.
47147
@Ignore
48148
@Test
49149
public void testExpandOnArray() throws Exception {
@@ -63,6 +163,117 @@ public void testExpandWithAlias() throws Exception {
63163
schema("age", "bigint"),
64164
schema("id", "bigint"),
65165
schema("addr", "struct"));
166+
verifyDataRows(
167+
response,
168+
rows(
169+
"abbas",
170+
null,
171+
24,
172+
new JSONObject()
173+
.put("city", "New york city")
174+
.put("state", "NY")
175+
.put("moveInDate", new JSONObject().put("dateAndTime", "1984-04-12 09:07:42"))),
176+
rows(
177+
"abbas",
178+
null,
179+
24,
180+
new JSONObject()
181+
.put("city", "bellevue")
182+
.put("state", "WA")
183+
.put(
184+
"moveInDate",
185+
new JSONArray()
186+
.put(new JSONObject().put("dateAndTime", "2023-05-03 08:07:42"))
187+
.put(new JSONObject().put("dateAndTime", "2001-11-11 04:07:44")))),
188+
rows(
189+
"abbas",
190+
null,
191+
24,
192+
new JSONObject()
193+
.put("city", "seattle")
194+
.put("state", "WA")
195+
.put("moveInDate", new JSONObject().put("dateAndTime", "1966-03-19 03:04:55"))),
196+
rows(
197+
"abbas",
198+
null,
199+
24,
200+
new JSONObject()
201+
.put("city", "chicago")
202+
.put("state", "IL")
203+
.put("moveInDate", new JSONObject().put("dateAndTime", "2011-06-01 01:01:42"))),
204+
rows(
205+
"chen",
206+
null,
207+
32,
208+
new JSONObject()
209+
.put("city", "Miami")
210+
.put("state", "Florida")
211+
.put("moveInDate", new JSONObject().put("dateAndTime", "1901-08-11 04:03:33"))),
212+
rows(
213+
"chen",
214+
null,
215+
32,
216+
new JSONObject()
217+
.put("city", "los angeles")
218+
.put("state", "CA")
219+
.put("moveInDate", new JSONObject().put("dateAndTime", "2023-05-03 08:07:42"))),
220+
rows(
221+
"peng",
222+
null,
223+
26,
224+
new JSONObject()
225+
.put("city", "san diego")
226+
.put("state", "CA")
227+
.put("moveInDate", new JSONObject().put("dateAndTime", "2001-11-11 04:07:44"))),
228+
rows(
229+
"peng",
230+
null,
231+
26,
232+
new JSONObject()
233+
.put("city", "austin")
234+
.put("state", "TX")
235+
.put("moveInDate", new JSONObject().put("dateAndTime", "1977-07-13 09:04:41"))),
236+
rows(
237+
"andy",
238+
4,
239+
19,
240+
new JSONObject()
241+
.put("city", "houston")
242+
.put("state", "TX")
243+
.put("moveInDate", new JSONObject().put("dateAndTime", "1933-12-12 05:05:45"))),
244+
rows(
245+
"david",
246+
null,
247+
25,
248+
new JSONObject()
249+
.put("city", "raleigh")
250+
.put("state", "NC")
251+
.put("moveInDate", new JSONObject().put("dateAndTime", "1909-06-17 01:04:21"))),
252+
rows(
253+
"david",
254+
null,
255+
25,
256+
new JSONObject()
257+
.put("city", "charlotte")
258+
.put("state", "SC")
259+
.put(
260+
"moveInDate",
261+
new JSONArray()
262+
.put(new JSONObject().put("dateAndTime", "2001-11-11 04:07:44")))));
263+
}
264+
265+
@Test
266+
public void testExpandWithEval() throws Exception {
267+
JSONObject response =
268+
executeQuery(
269+
String.format("source=%s | eval addr=address | expand addr", TEST_INDEX_NESTED_SIMPLE));
270+
verifySchema(
271+
response,
272+
schema("name", "string"),
273+
schema("age", "bigint"),
274+
schema("address", "array"),
275+
schema("id", "bigint"),
276+
schema("addr", "struct"));
66277
verifyNumOfRows(response, 11);
67278
}
68279
}

‎opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchExecutionEngine.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ private void buildResultSet(
241241
if (!values.isEmpty()) {
242242
exprType = values.getFirst().tupleValue().get(columnName).type();
243243
} else {
244+
// Using UNDEFINED instead of UNKNOWN to avoid throwing exception
244245
exprType = ExprCoreType.UNDEFINED;
245246
}
246247
} else {

0 commit comments

Comments
 (0)