Stabilize subquery PPL ITs on the analytics-engine route (#5555)

ahkcs · web-flow · commit 3829dfa4257d · 2026-06-16T13:32:54.000-07:00
CalcitePPLScalarSubqueryIT and CalcitePPLInSubqueryIT both seed an extra worker doc via an unconditional raw PUT in init(). init() runs as @before before every test method, and the analytics-engine parquet-backed store is append-only on same-_id PUT, so the doc accumulated a duplicate per method and inflated row counts across the suite. Guard the seed on a pre-loadIndex isIndexExist check so it runs exactly once; behavior is unchanged on the v2/Calcite route (same end state). Skip the four tests that exercise behaviors the analytics-engine route does not support, using the assumeNotAnalytics(...) registry (AnalyticsRouteLimitation) plus matching excludeTestsMatching entries in integTestRemote so the skip set stays countable in one place. Two new AnalyticsRouteLimitation constants: - TEXT_FIELD_EXACT_MATCH: exact = / == on an explicitly text-mapped field (no .keyword sub-field) returns no rows on the DataFusion scan. Sibling of DYNAMIC_STRING_NO_KEYWORD for explicitly-mapped (not dynamic) text fields. Covers testTwoUncorrelatedScalarSubqueriesInOr, testInSubqueryWithTableAlias, testInCorrelatedSubquery (each filters department/occupation = '...'). - SUBSEARCH_MAXOUT_IN_SUBQUERY: the subsearch.maxout cap is lowered as a LIMIT on the in-subquery semi-join's right side, which the route does not honor, so the subsearch returns all rows. Covers testSubsearchMaxOut. Results (-Dtests.analytics.parquet_indices=true against the analytics route): CalcitePPLScalarSubqueryIT: 2/14 -> 13/13 run, 0 fail (1 excluded) CalcitePPLInSubqueryIT: 7/18 -> 14/15 run, 0 fail (3 excluded; 1 @ignore) v2/Calcite route unchanged: 14/14 and 17/17 (1 pre-existing @ignore). Signed-off-by: Kai Huang <ahkcs@amazon.com>
diff --git a/integ-test/build.gradle b/integ-test/build.gradle
@@ -1135,6 +1135,18 @@ task integTestRemote(type: RestIntegTestTask) {
             //    nullable tiebreak these would need orders nulls differently than v2/Calcite.
             excludeTestsMatching '*CalcitePPLEnhancedCoalesceIT.testCoalesceWithMixedTypes'
             excludeTestsMatching '*CalcitePPLEnhancedCoalesceIT.testCoalesceBasic'
+
+            // === Excludes: CalcitePPLScalarSubqueryIT / CalcitePPLInSubqueryIT route divergences ===
+            // Each test also carries an in-test assumeNotAnalytics(...) recording the reason (see
+            // AnalyticsRouteLimitation); listed here so the AE-route skip set stays countable.
+            //  - Exact equality on an explicitly text-mapped field (department/occupation = '...')
+            //    in the subsearch returns no rows on the AE route (analyzed text, no .keyword).
+            excludeTestsMatching '*CalcitePPLScalarSubqueryIT.testTwoUncorrelatedScalarSubqueriesInOr'
+            excludeTestsMatching '*CalcitePPLInSubqueryIT.testInSubqueryWithTableAlias'
+            excludeTestsMatching '*CalcitePPLInSubqueryIT.testInCorrelatedSubquery'
+            //  - subsearch.maxout is lowered as a LIMIT on the in-subquery semi-join's right side,
+            //    which the AE route does not honor, so the subsearch returns all rows.
+            excludeTestsMatching '*CalcitePPLInSubqueryIT.testSubsearchMaxOut'
         }
     }
 
diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLInSubqueryIT.java
@@ -5,9 +5,12 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.legacy.TestUtils.isIndexExist;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OCCUPATION;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORKER;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORK_INFORMATION;
+import static org.opensearch.sql.util.AnalyticsRouteLimitation.SUBSEARCH_MAXOUT_IN_SUBQUERY;
+import static org.opensearch.sql.util.AnalyticsRouteLimitation.TEXT_FIELD_EXACT_MATCH;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
@@ -31,16 +34,22 @@ public void init() throws Exception {
     super.init();
     enableCalcite();
 
+    // init() runs as @Before, before every test method. On the analytics route the parquet-backed
+    // store is append-only on same-_id PUT, so seed the extra worker doc only when the index is
+    // first created — otherwise it accumulates a duplicate per test method and inflates row counts.
+    boolean workerExisted = isIndexExist(client(), TEST_INDEX_WORKER);
     loadIndex(Index.WORKER);
     loadIndex(Index.WORK_INFORMATION);
     loadIndex(Index.OCCUPATION);
 
-    // {"index":{"_id":"7"}}
-    // {"id":1006,"name":"Tommy","occupation":"Teacher","country":"USA","salary":30000}
-    Request request1 = new Request("PUT", "/" + TEST_INDEX_WORKER + "/_doc/7?refresh=true");
-    request1.setJsonEntity(
-        "{\"id\":1006,\"name\":\"Tommy\",\"occupation\":\"Teacher\",\"country\":\"USA\",\"salary\":30000}");
-    client().performRequest(request1);
+    if (!workerExisted) {
+      // {"index":{"_id":"7"}}
+      // {"id":1006,"name":"Tommy","occupation":"Teacher","country":"USA","salary":30000}
+      Request request1 = new Request("PUT", "/" + TEST_INDEX_WORKER + "/_doc/7?refresh=true");
+      request1.setJsonEntity(
+          "{\"id\":1006,\"name\":\"Tommy\",\"occupation\":\"Teacher\",\"country\":\"USA\",\"salary\":30000}");
+      client().performRequest(request1);
+    }
   }
 
   @Test
@@ -340,6 +349,8 @@ public void failWhenNumOfColumnsNotMatchOutputOfSubquery() {
 
   @Test
   public void testInSubqueryWithTableAlias() throws IOException {
+    // Subsearch filters a text-mapped field with exact equality (i.department = 'DATA').
+    assumeNotAnalytics(TEXT_FIELD_EXACT_MATCH);
     JSONObject result =
         executeQuery(
             String.format(
@@ -358,6 +369,8 @@ public void testInSubqueryWithTableAlias() throws IOException {
 
   @Test
   public void testInCorrelatedSubquery() throws IOException {
+    // Subsearch filters a text-mapped field with exact equality (occupation = 'Engineer').
+    assumeNotAnalytics(TEXT_FIELD_EXACT_MATCH);
     JSONObject result =
         executeQuery(
             String.format(
@@ -372,6 +385,7 @@ public void testInCorrelatedSubquery() throws IOException {
 
   @Test
   public void testSubsearchMaxOut() throws IOException {
+    assumeNotAnalytics(SUBSEARCH_MAXOUT_IN_SUBQUERY);
     setSubsearchMaxOut(1);
     JSONObject result =
         executeQuery(
diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLScalarSubqueryIT.java
@@ -5,9 +5,11 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.legacy.TestUtils.isIndexExist;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OCCUPATION;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORKER;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WORK_INFORMATION;
+import static org.opensearch.sql.util.AnalyticsRouteLimitation.TEXT_FIELD_EXACT_MATCH;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
@@ -27,16 +29,22 @@ public void init() throws Exception {
     super.init();
     enableCalcite();
 
+    // init() runs as @Before, before every test method. On the analytics route the parquet-backed
+    // store is append-only on same-_id PUT, so seed the extra worker doc only when the index is
+    // first created — otherwise it accumulates a duplicate per test method and inflates row counts.
+    boolean workerExisted = isIndexExist(client(), TEST_INDEX_WORKER);
     loadIndex(Index.WORKER);
     loadIndex(Index.WORK_INFORMATION);
     loadIndex(Index.OCCUPATION);
 
-    // {"index":{"_id":"7"}}
-    // {"id":1006,"name":"Tommy","occupation":"Teacher","country":"USA","salary":30000}
-    Request request1 = new Request("PUT", "/" + TEST_INDEX_WORKER + "/_doc/7?refresh=true");
-    request1.setJsonEntity(
-        "{\"id\":1006,\"name\":\"Tommy\",\"occupation\":\"Teacher\",\"country\":\"USA\",\"salary\":30000}");
-    client().performRequest(request1);
+    if (!workerExisted) {
+      // {"index":{"_id":"7"}}
+      // {"id":1006,"name":"Tommy","occupation":"Teacher","country":"USA","salary":30000}
+      Request request1 = new Request("PUT", "/" + TEST_INDEX_WORKER + "/_doc/7?refresh=true");
+      request1.setJsonEntity(
+          "{\"id\":1006,\"name\":\"Tommy\",\"occupation\":\"Teacher\",\"country\":\"USA\",\"salary\":30000}");
+      client().performRequest(request1);
+    }
   }
 
   @Test
@@ -230,6 +238,8 @@ public void testDisjunctiveCorrelatedScalarSubquery() throws IOException {
 
   @Test
   public void testTwoUncorrelatedScalarSubqueriesInOr() throws IOException {
+    // Subsearch filters a text-mapped field with exact equality (department = 'DATA').
+    assumeNotAnalytics(TEXT_FIELD_EXACT_MATCH);
     JSONObject result =
         executeQuery(
             String.format(
diff --git a/integ-test/src/test/java/org/opensearch/sql/util/AnalyticsRouteLimitation.java b/integ-test/src/test/java/org/opensearch/sql/util/AnalyticsRouteLimitation.java
@@ -47,6 +47,19 @@ public enum AnalyticsRouteLimitation {
           + " sub-field that standard OpenSearch adds, and the DataFusion scan can't match on an"
           + " analyzed text field."),
 
+  /**
+   * Exact equality ({@code =} / {@code ==}) on an explicitly {@code text}-mapped field (no {@code
+   * .keyword} sub-field) returns no rows on the analytics-engine route — the DataFusion scan can't
+   * match on an analyzed text field. The sibling of {@link #DYNAMIC_STRING_NO_KEYWORD} for fields
+   * that are mapped {@code text} on purpose rather than by dynamic mapping. Verified directly:
+   * {@code where department = 'DATA'} returns no rows while {@code like(department, 'DATA')} and
+   * keyword-field equality both work.
+   */
+  TEXT_FIELD_EXACT_MATCH(
+      "Exact equality (= / ==) on an explicitly text-mapped field (no .keyword sub-field) returns"
+          + " no rows on the analytics-engine route: the DataFusion scan can't match on an analyzed"
+          + " text field. Use like() or a keyword field instead."),
+
   /**
    * The analytics-engine storage path ({@code DataFormatAwareEngine}) does not support in-place
    * document mutation, so tests that seed state via raw {@code PUT}+{@code DELETE} can't run on
@@ -108,7 +121,19 @@ public enum AnalyticsRouteLimitation {
   HEAD_WITHOUT_STABLE_SORT(
       "head N without a sort on a key that is unique over the head window is non-deterministic on"
           + " the analytics-engine route, and a nullable tiebreak orders nulls differently than the"
-          + " v2/Calcite path.");
+          + " v2/Calcite path."),
+
+  /**
+   * The {@code subsearch.maxout} cap on an {@code in}-subquery is lowered as a {@code LIMIT} on the
+   * right-hand side of the semi-join ({@code LogicalSystemLimit(fetch=N, type=SUBSEARCH_MAXOUT)}).
+   * The analytics-engine route does not honor that LIMIT, so the subsearch returns all rows
+   * regardless of the cap. Verified: with {@code subsearch.maxout=1} an {@code id in [...]}
+   * subquery still returns every matching row.
+   */
+  SUBSEARCH_MAXOUT_IN_SUBQUERY(
+      "subsearch.maxout is not honored on the analytics-engine route: the LIMIT lowered onto the"
+          + " in-subquery semi-join's right side is dropped, so the subsearch returns all rows"
+          + " regardless of the cap.");
 
   private final String reason;
 
