Test filter on both nested and root level fields

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -1785,11 +1785,22 @@ public void testInternalItemAccessOnStructs() throws IOException {
   @Test
   public void testFilterOnNestedFields() throws IOException {
     assertYamlEqualsIgnoreId(
-        loadExpectedPlan("filter_on_nested.yaml"),
+        loadExpectedPlan("filter_nested.yaml"),
         explainQueryYaml(
             StringUtils.format(
                 "source=%s | eval proj_name_len=length(projects.name) | fields projects.name,"
                     + " proj_name_len | where proj_name_len > 29",
                 TEST_INDEX_DEEP_NESTED)));
   }
+
+  @Test
+  public void testFilterOnNestedAndRootFields() throws IOException {
+    assertYamlEqualsIgnoreId(
+        loadExpectedPlan("filter_root_and_nested.yaml"),
+        // city.name is not in a nested object
+        explainQueryYaml(
+            StringUtils.format(
+                "source=%s | where city.name = 'Seattle' and length(projects.name) > 29",
+                TEST_INDEX_DEEP_NESTED)));
+  }
 }

…ctedOutput/calcite/filter_on_nested.yaml …xpectedOutput/calcite/filter_nested.yamlinteg-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml integ-test/src/test/resources/expectedOutput/calcite/filter_on_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite/filter_nested.yaml

@@ -0,0 +1,8 @@
+calcite:
+  logical: |
+    LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
+      LogicalProject(accounts=[$0], projects=[$2], city=[$4], account=[$8])
+        LogicalFilter(condition=[AND(=($7, 'Seattle'), >(CHAR_LENGTH($3), 29))])
+          CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]])
+  physical: |
+    CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]], PushDownContext=[[PROJECT->[accounts, projects, projects.name, city, city.name, account], SCRIPT->AND(=($4, 'Seattle'), >(CHAR_LENGTH($2), 29)), PROJECT->[accounts, projects, city, account], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"city.name":{"value":"Seattle","boost":1.0}}},{"nested":{"query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["projects.name",29]}},"boost":1.0}},"path":"projects","ignore_unmapped":false,"score_mode":"none","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["accounts","projects","city","account"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])

integ-test/src/test/resources/expectedOutput/calcite/filter_root_and_nested.yaml

@@ -0,0 +1,10 @@
+calcite:
+  logical: |
+    LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])
+      LogicalProject(accounts=[$0], projects=[$2], city=[$4], account=[$8])
+        LogicalFilter(condition=[AND(=($7, 'Seattle'), >(CHAR_LENGTH($3), 29))])
+          CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]])
+  physical: |
+    EnumerableLimit(fetch=[10000])
+      EnumerableCalc(expr#0..15=[{inputs}], expr#16=['Seattle':VARCHAR], expr#17=[=($t7, $t16)], expr#18=[CHAR_LENGTH($t3)], expr#19=[29], expr#20=[>($t18, $t19)], expr#21=[AND($t17, $t20)], accounts=[$t0], projects=[$t2], city=[$t4], account=[$t8], $condition=[$t21])
+        CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_deep_nested]])

…alcite_no_pushdown/filter_on_nested.yaml …t/calcite_no_pushdown/filter_nested.yamlinteg-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_on_nested.yaml renamed to integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/filter_nested.yaml

@@ -92,3 +92,19 @@ setup:
   - match: { total: 2 }
   - match: {"schema": [{"name": "id", "type": "string"}, {"name": "name", "type": "string"}, {"name": "NameLen", "type": "int"}]}
   - match: {"datarows": [["order2", "banana", 6], ["order3", "orange", 6]]}
+
+---
+"filter on both nested and root level fields ":
+  - skip:
+      features:
+        - headers
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test_nested_eval_filter | eval NameLen=LENGTH(items.name) | where NameLen> 5 and id = 'order2' | fields id, items.name, NameLen
+
+  - match: { total: 1 }
+  - match: {"schema": [{"name": "id", "type": "string"}, {"name": "items.name", "type": "string"}, {"name": "NameLen", "type": "int"}]}
+  - match: {"datarows": [["order2", "banana", 6]]}