Stabilize PPL ITs on the analytics-engine route (array/map-path/datatype/basic)

integ-test/build.gradle

@@ -1080,6 +1080,12 @@ task integTestRemote(type: RestIntegTestTask) {
             excludeTestsMatching '*SystemFunctionIT.typeof_opensearch_types'
             // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`.
             excludeTestsMatching '*DataTypeIT.test_alias_data_type'
+            // scaled_float reports bigint (not double); string->numeric coerces to null (not 0);
+            // cross-index incompatible field types are rejected; boolean-from-string test DELETEs.
+            excludeTestsMatching '*DataTypeIT.test_numeric_data_types'
+            excludeTestsMatching '*DataTypeIT.testNumericFieldFromString'
+            excludeTestsMatching '*DataTypeIT.testBooleanFieldFromNumberAcrossWildcardIndices'
+            excludeTestsMatching '*DataTypeIT.testBooleanFieldFromString'
             // CalciteAliasFieldAggregationIT: raw-PUT alias index can't be created on the AE route
             // and every test queries alias fields directly — whole class doomed.
             excludeTestsMatching 'org.opensearch.sql.calcite.remote.CalciteAliasFieldAggregationIT'
@@ -1204,6 +1210,35 @@ task integTestRemote(type: RestIntegTestTask) {
             // === Excludes: CalcitePPLAppendPipeCommandIT route divergence ===
             // appendpipe drops the main pipeline's rows on AE (subpipe filter applied in place).
             excludeTestsMatching '*CalcitePPLAppendPipeCommandIT.testDoubleAppendPipeWithFilter'
+
+            // === Excludes: CalciteArrayFunctionIT route divergences ===
+            // Higher-order array functions (transform/mvmap, reduce, filter, exists, forall) take a
+            // PPL lambda the AE backends can't execute ('No backend supports scalar function [...]').
+            excludeTestsMatching '*CalciteArrayFunctionIT.testForAll'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testExists'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testFilter'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testTransform'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForTwoInput'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForWithDouble'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testTransformForWithUDF'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testReduce'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testReduce2'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testReduce3'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testReduceWithUDF'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testMvmap'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithAddition'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithEvalFieldReference'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithNestedFunction'
+            excludeTestsMatching '*CalciteArrayFunctionIT.testMvmapWithOtherFieldReference'
+
+            // === Excludes: CalcitePPLMapPathIT route divergences ===
+            // mvcombine -> ARRAY_AGG (no such AggregateFunction enum); addtotals -> DataFusion join panic.
+            excludeTestsMatching '*CalcitePPLMapPathIT.testMvcombineOnMapPath'
+            excludeTestsMatching '*CalcitePPLMapPathIT.testAddtotalsOnMapPath'
+
+            // === Excludes: CalcitePPLBasicIT route divergence ===
+            // REGEXP filter throws a backend NullPointerException on the AE route.
+            excludeTestsMatching '*CalcitePPLBasicIT.testRegexpFilter'
         }
     }
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java

@@ -6,6 +6,7 @@
 package org.opensearch.sql.calcite.remote;
 
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
+import static org.opensearch.sql.util.Capability.ARRAY_HIGHER_ORDER_FUNC;
 import static org.opensearch.sql.util.MatcherUtils.*;
 
 import java.io.IOException;
@@ -15,14 +16,19 @@
 import org.junit.jupiter.api.Test;
 import org.opensearch.client.ResponseException;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
+import org.opensearch.sql.util.RequiresCapability;
 
 public class CalciteArrayFunctionIT extends PPLIntegTestCase {
   @Override
   public void init() throws Exception {
     super.init();
     enableCalcite();
     loadIndex(Index.BANK);
-    loadIndex(Index.ARRAY);
+    // No test queries the array index (all build arrays inline via array()); its multi-value
+    // numbers field can't be bulk-loaded into the parquet store, so skip it on the AE route.
+    if (!isAnalyticsParquetIndicesEnabled()) {
+      loadIndex(Index.ARRAY);
+    }
   }
 
   @Test
@@ -85,6 +91,11 @@ public void testArrayLength() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testForAll() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -99,6 +110,11 @@ public void testForAll() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testExists() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -113,6 +129,11 @@ public void testExists() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testFilter() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -127,6 +148,11 @@ public void testFilter() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testTransform() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -141,6 +167,11 @@ public void testTransform() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testTransformForTwoInput() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -155,6 +186,11 @@ public void testTransformForTwoInput() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testTransformForWithDouble() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -169,6 +205,11 @@ public void testTransformForWithDouble() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testTransformForWithUDF() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -185,6 +226,11 @@ public void testTransformForWithUDF() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testReduce() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -202,6 +248,11 @@ public void testReduce() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testReduce2() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -216,6 +267,11 @@ public void testReduce2() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testReduce3() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -231,6 +287,11 @@ public void testReduce3() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testReduceWithUDF() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -796,6 +857,11 @@ public void testSplitWithEmptyDelimiter() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testMvmap() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -809,6 +875,11 @@ public void testMvmap() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testMvmapWithAddition() throws IOException {
     JSONObject actual =
         executeQuery(
@@ -822,6 +893,11 @@ public void testMvmapWithAddition() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testMvmapWithNestedFunction() throws IOException {
     // Test mvmap with mvindex as first argument - extracts field name from nested function
     // Equivalent to Splunk: mvmap(mvindex(arr, 1, 3), arr * 10)
@@ -839,6 +915,11 @@ public void testMvmapWithNestedFunction() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testMvmapWithOtherFieldReference() throws IOException {
     // Test mvmap with reference to another field in the expression
     // The first record in bank has age=32, so array(1,2,3) * 32 = [32, 64, 96]
@@ -854,6 +935,11 @@ public void testMvmapWithOtherFieldReference() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ARRAY_HIGHER_ORDER_FUNC,
+      note =
+          "Higher-order array function (transform/mvmap/reduce/filter/exists/forall) takes a"
+              + " lambda.")
   public void testMvmapWithEvalFieldReference() throws IOException {
     // Test mvmap with reference to another field created by eval
     // array(1,2,3) * 10 = [10, 20, 30]

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java

@@ -8,6 +8,7 @@
 import static org.junit.Assume.assumeFalse;
 import static org.opensearch.sql.legacy.TestUtils.isIndexExist;
 import static org.opensearch.sql.legacy.TestsConstants.*;
+import static org.opensearch.sql.util.Capability.REGEXP_FILTER;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
@@ -21,6 +22,7 @@
 import org.opensearch.client.ResponseException;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
+import org.opensearch.sql.util.RequiresCapability;
 
 public class CalcitePPLBasicIT extends PPLIntegTestCase {
 
@@ -154,6 +156,9 @@ public void testFilterQuery4() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = REGEXP_FILTER,
+      note = "REGEXP filter throws a backend NullPointerException on the AE route.")
   public void testRegexpFilter() throws IOException {
     JSONObject actual = executeQuery("source=test | where name REGEXP 'he.*' | fields name, age");
     verifySchema(actual, schema("name", "string"), schema("age", "bigint"));

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLMapPathIT.java

@@ -5,6 +5,8 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.util.Capability.ADDTOTALS_JOIN_PANIC;
+import static org.opensearch.sql.util.Capability.MVCOMBINE_ARRAY_AGG;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
@@ -22,6 +24,7 @@
 import org.junit.jupiter.api.Test;
 import org.opensearch.client.Request;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
+import org.opensearch.sql.util.RequiresCapability;
 
 /**
  * Integration tests for PPL queries that reference MAP dotted paths (e.g. {@code doc.user.name}).
@@ -141,6 +144,9 @@ public void testFieldsExclusionOnMapPath() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = ADDTOTALS_JOIN_PANIC,
+      note = "addtotals crashes the DataFusion backend with a join panic on the AE route.")
   public void testAddtotalsOnMapPath() throws IOException {
     JSONObject result =
         ppl(
@@ -163,6 +169,9 @@ public void testAddtotalsOnMapPath() throws IOException {
   }
 
   @Test
+  @RequiresCapability(
+      value = MVCOMBINE_ARRAY_AGG,
+      note = "mvcombine lowers to ARRAY_AGG, unregistered on the analytics backend.")
   public void testMvcombineOnMapPath() throws IOException {
     JSONObject result =
         ppl(