Skip to content

Commit 8bf28be

Browse files
yuancuyuancu
committed
Add integration tests for filtering on nested
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent 18a8431 commit 8bf28be

2 files changed

Lines changed: 96 additions & 2 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,18 @@
55

66
package org.opensearch.sql.calcite.remote;
77

8+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CASCADED_NESTED;
9+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED;
10+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE;
11+
import static org.opensearch.sql.util.MatcherUtils.rows;
12+
import static org.opensearch.sql.util.MatcherUtils.schema;
13+
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
14+
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
15+
16+
import java.io.IOException;
17+
import java.util.List;
18+
import java.util.Map;
19+
import org.json.JSONObject;
820
import org.junit.Test;
921
import org.opensearch.sql.ppl.WhereCommandIT;
1022

@@ -14,6 +26,7 @@ public void init() throws Exception {
1426
super.init();
1527
enableCalcite();
1628
loadIndex(Index.NESTED_SIMPLE);
29+
loadIndex(Index.DEEP_NESTED);
1730
loadIndex(Index.CASCADED_NESTED);
1831
}
1932

@@ -24,5 +37,86 @@ protected String getIncompatibleTypeErrMsg() {
2437
}
2538

2639
@Test
27-
public void testWhereOnNestedField() {}
40+
public void testFilterOnComputedNestedFields() throws IOException {
41+
JSONObject result =
42+
executeQuery(
43+
String.format(
44+
"source=%s | eval proj_name_len=length(projects.name) | fields projects.name,"
45+
+ " proj_name_len | where proj_name_len > 29",
46+
TEST_INDEX_DEEP_NESTED));
47+
verifySchema(result, schema("projects.name", "string"), schema("proj_name_len", "int"));
48+
verifyDataRows(result, rows("AWS Redshift Spectrum querying", 30));
49+
}
50+
51+
@Test
52+
public void testFilterOnNestedAndRootFields() throws IOException {
53+
JSONObject result =
54+
executeQuery(
55+
String.format(
56+
"source=%s | where city.name = 'Seattle' and length(projects.name) > 29 | fields"
57+
+ " city.name, projects.name",
58+
TEST_INDEX_DEEP_NESTED));
59+
verifySchema(result, schema("city.name", "string"), schema("projects.name", "string"));
60+
verifyDataRows(result, rows("Seattle", "AWS Redshift Spectrum querying"));
61+
}
62+
63+
@Test
64+
public void testFilterOnNestedFields() throws IOException {
65+
// address is a nested object
66+
JSONObject result1 =
67+
executeQuery(
68+
String.format(
69+
"source=%s | where address.city = 'New york city' | fields address.city",
70+
TEST_INDEX_NESTED_SIMPLE));
71+
verifySchema(result1, schema("address.city", "string"));
72+
verifyDataRows(result1, rows("New york city"));
73+
74+
JSONObject result2 =
75+
executeQuery(
76+
String.format(
77+
"source=%s | where address.city in ('Miami', 'san diego') | fields address.city",
78+
TEST_INDEX_NESTED_SIMPLE));
79+
verifyDataRows(result2, rows("Miami"), rows("san diego"));
80+
}
81+
82+
@Test
83+
public void testFilterOnMultipleCascadedNestedFields() throws IOException {
84+
JSONObject result =
85+
executeQuery(
86+
String.format(
87+
"source=%s | where author.books.reviews.rating >=4 and author.books.reviews.rating"
88+
+ " < 6 and author.books.title = 'The Shining' | fields author.books",
89+
TEST_INDEX_CASCADED_NESTED));
90+
verifySchema(result, schema("author.books", "array"));
91+
verifyDataRows(
92+
result,
93+
rows(
94+
List.of(
95+
Map.of(
96+
"title",
97+
"The Shining",
98+
"reviews",
99+
List.of(
100+
Map.of(
101+
"review_date",
102+
"2022-09-03",
103+
"rating",
104+
3,
105+
"comment",
106+
"Brilliant but terrifying"),
107+
Map.of(
108+
"review_date",
109+
"2023-04-12",
110+
"rating",
111+
4,
112+
"comment",
113+
"Psychological horror at its best"),
114+
Map.of(
115+
"review_date",
116+
"2023-10-28",
117+
"rating",
118+
2,
119+
"comment",
120+
"Too slow in places"))))));
121+
}
28122
}

‎integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ calcite:
55
LogicalFilter(condition=[AND(SEARCH($4, Sarg[[4..6)]), =($6, 'The Shining'))])
66
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]])
77
physical: |
8-
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},{"term":{"author.books.title.keyword":{"value":"The Shining","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])
8+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"nested":{"query":{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},"path":"author.books.reviews","ignore_unmapped":false,"score_mode":"none","boost":1.0}},{"nested":{"query":{"term":{"author.books.title":{"value":"The Shining","boost":1.0}}},"path":"author.books","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])

0 commit comments

Comments
 (0)