Skip to content

Commit 1ed9eee

Browse files
yuancuyuancu
committed
Support nested queries for simple queries: range, term, terms, etc
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent 3a13eb6 commit 1ed9eee

15 files changed

Lines changed: 210 additions & 54 deletions

File tree

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT;
99
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
1010
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES;
11+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CASCADED_NESTED;
1112
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DEEP_NESTED;
1213
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS;
1314
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE;
@@ -45,6 +46,7 @@ public void init() throws Exception {
4546
loadIndex(Index.WORK_INFORMATION);
4647
loadIndex(Index.WEBLOG);
4748
loadIndex(Index.DEEP_NESTED);
49+
loadIndex(Index.CASCADED_NESTED);
4850
}
4951

5052
@Override
@@ -1783,9 +1785,9 @@ public void testInternalItemAccessOnStructs() throws IOException {
17831785
}
17841786

17851787
@Test
1786-
public void testFilterOnNestedFields() throws IOException {
1788+
public void testFilterOnComputedNestedFields() throws IOException {
17871789
assertYamlEqualsIgnoreId(
1788-
loadExpectedPlan("filter_nested.yaml"),
1790+
loadExpectedPlan("filter_computed_nested.yaml"),
17891791
explainQueryYaml(
17901792
StringUtils.format(
17911793
"source=%s | eval proj_name_len=length(projects.name) | fields projects.name,"
@@ -1797,10 +1799,41 @@ public void testFilterOnNestedFields() throws IOException {
17971799
public void testFilterOnNestedAndRootFields() throws IOException {
17981800
assertYamlEqualsIgnoreId(
17991801
loadExpectedPlan("filter_root_and_nested.yaml"),
1800-
// city.name is not in a nested object
1802+
// city is not in a nested object
18011803
explainQueryYaml(
18021804
StringUtils.format(
18031805
"source=%s | where city.name = 'Seattle' and length(projects.name) > 29",
18041806
TEST_INDEX_DEEP_NESTED)));
18051807
}
1808+
1809+
@Test
1810+
public void testFilterOnNestedFields() throws IOException {
1811+
assertYamlEqualsIgnoreId(
1812+
loadExpectedPlan("filter_nested_term.yaml"),
1813+
// address is a nested object
1814+
explainQueryYaml(
1815+
StringUtils.format(
1816+
"source=%s | where address.city = 'New york city'", TEST_INDEX_NESTED_SIMPLE)));
1817+
1818+
assertYamlEqualsIgnoreId(
1819+
loadExpectedPlan("filter_nested_terms.yaml"),
1820+
explainQueryYaml(
1821+
StringUtils.format(
1822+
"source=%s | where address.city in ('Miami', 'san diego')",
1823+
TEST_INDEX_NESTED_SIMPLE)));
1824+
}
1825+
1826+
@Test
1827+
public void testFilterOnMultipleCascadedNestedFields() throws IOException {
1828+
// 1. Access two different hierarchies of nested fields, one at author.books.reviews, another at
1829+
// author.books
1830+
// 2. One is pushed as nested range query, another is pushed as nested filter query.
1831+
assertYamlEqualsIgnoreId(
1832+
loadExpectedPlan("filter_multiple_nested_cascaded_range.yaml"),
1833+
explainQueryYaml(
1834+
StringUtils.format(
1835+
"source=%s | where author.books.reviews.rating >=4 and author.books.reviews.rating"
1836+
+ " < 6 and author.books.title = 'The Shining'",
1837+
TEST_INDEX_CASCADED_NESTED)));
1838+
}
18061839
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteWhereCommandIT.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,24 @@
55

66
package org.opensearch.sql.calcite.remote;
77

8+
import org.junit.Test;
89
import org.opensearch.sql.ppl.WhereCommandIT;
910

1011
public class CalciteWhereCommandIT extends WhereCommandIT {
1112
@Override
1213
public void init() throws Exception {
1314
super.init();
1415
enableCalcite();
16+
loadIndex(Index.NESTED_SIMPLE);
17+
loadIndex(Index.CASCADED_NESTED);
1518
}
1619

1720
@Override
1821
protected String getIncompatibleTypeErrMsg() {
1922
return "In expression types are incompatible: fields type LONG, values type [INTEGER, INTEGER,"
2023
+ " STRING]";
2124
}
25+
26+
@Test
27+
public void testWhereOnNestedField() {}
2228
}

‎integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -691,6 +691,11 @@ public enum Index {
691691
"_doc",
692692
getDeepNestedIndexMapping(),
693693
"src/test/resources/deep_nested_index_data.json"),
694+
CASCADED_NESTED(
695+
TestsConstants.TEST_INDEX_CASCADED_NESTED,
696+
"_doc",
697+
getMappingFile("cascaded_nested_index_mapping.json"),
698+
"src/test/resources/cascaded_nested.json"),
694699
TELEMETRY(
695700
TestsConstants.TEST_INDEX_TELEMETRY,
696701
"_doc",

‎integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ public class TestsConstants {
5454
public static final String TEST_INDEX_TIME_DATA = TEST_INDEX + "_time_data";
5555

5656
public static final String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested";
57+
public static final String TEST_INDEX_CASCADED_NESTED = TEST_INDEX + "_cascaded_nested";
5758
public static final String TEST_INDEX_TELEMETRY = TEST_INDEX + "_telemetry";
5859
public static final String TEST_INDEX_STRINGS = TEST_INDEX + "_strings";
5960
public static final String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric";

‎integ-test/src/test/resources/cascaded_nested.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{"index": {"_id": "1"}}
2+
{"author": {"name": "J.K. Rowling", "books": [{"title": "Harry Potter and the Sorcerer's Stone", "reviews": [{"rating": 5, "comment": "Magical and enchanting!", "review_date": "2023-01-15"}, {"rating": 4, "comment": "Great for kids and adults", "review_date": "2023-06-22"}]}, {"title": "Harry Potter and the Chamber of Secrets", "reviews": [{"rating": 5, "comment": "Even better than the first", "review_date": "2023-02-10"}, {"rating": 4, "comment": "Darker tone emerging", "review_date": "2023-07-18"}]}]}}
3+
{"index": {"_id": "2"}}
4+
{"author": {"name": "George R.R. Martin", "books": [{"title": "A Game of Thrones", "reviews": [{"rating": 4, "comment": "Epic fantasy masterpiece", "review_date": "2022-11-05"}, {"rating": 3, "comment": "Too many characters to track", "review_date": "2023-03-20"}]}, {"title": "A Clash of Kings", "reviews": [{"rating": 2, "comment": "Incredible plot twists", "review_date": "2023-08-14"}]}]}}
5+
{"index": {"_id": "3"}}
6+
{"author": {"name": "Stephen King", "books": [{"title": "The Shining", "reviews": [{"rating": 3, "comment": "Brilliant but terrifying", "review_date": "2022-09-03"}, {"rating": 4, "comment": "Psychological horror at its best", "review_date": "2023-04-12"}, {"rating": 2, "comment": "Too slow in places", "review_date": "2023-10-28"}]}]}}

integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite/filter_computed_nested.yaml

File renamed without changes.

‎integ-test/src/test/resources/expectedOutput/calcite/filter_multiple_nested_cascaded_range.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
calcite:
2+
logical: |
3+
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
4+
LogicalProject(author=[$0])
5+
LogicalFilter(condition=[AND(SEARCH($4, Sarg[[4..6)]), =($6, 'The Shining'))])
6+
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]])
7+
physical: |
8+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_cascaded_nested]], PushDownContext=[[PROJECT->[author, author.books.reviews.rating, author.books.title], FILTER->AND(SEARCH($1, Sarg[[4..6)]), =($2, 'The Shining')), PROJECT->[author], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"author.books.reviews.rating":{"from":4.0,"to":6.0,"include_lower":true,"include_upper":false,"boost":1.0}}},{"term":{"author.books.title.keyword":{"value":"The Shining","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["author"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])

‎integ-test/src/test/resources/expectedOutput/calcite/filter_nested_term.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
calcite:
2+
logical: |
3+
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
4+
LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7])
5+
LogicalFilter(condition=[=($2, 'New york city')])
6+
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]])
7+
physical: |
8+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->=($2, 'New york city'), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"term":{"address.city.keyword":{"value":"New york city","boost":1.0}}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])

‎integ-test/src/test/resources/expectedOutput/calcite/filter_nested_terms.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
calcite:
2+
logical: |
3+
LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
4+
LogicalProject(name=[$0], address=[$1], id=[$6], age=[$7])
5+
LogicalFilter(condition=[SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR)])
6+
CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]])
7+
physical: |
8+
CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_nested_simple]], PushDownContext=[[PROJECT->[name, address, address.city, id, age], FILTER->SEARCH($2, Sarg['Miami':VARCHAR, 'san diego':VARCHAR]:VARCHAR), PROJECT->[name, address, id, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"nested":{"query":{"terms":{"address.city.keyword":["Miami","san diego"],"boost":1.0}},"path":"address","ignore_unmapped":false,"score_mode":"none","boost":1.0}},"_source":{"includes":["name","address","id","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])

integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_computed_nested.yaml

File renamed without changes.

0 commit comments

Comments
 (0)