Branch 7 stats tests on isAnalyticsParquetIndicesEnabled

When -Dtests.analytics.parquet_indices=true is set, every test-created
index is parquet-backed and RestUnifiedQueryAction.isAnalyticsIndex
routes all queries through the analytics-engine backend (DataFusion).
Seven tests need analytics-specific assertions because DataFusion follows
different null-bucket, SQL-spec, and TDigest interpolation semantics
than the legacy V2 / Calcite-DSL paths:

SQL-spec semantics (DataFusion follows the spec; legacy DSL returns 0):
- testSumWithNull — SUM of all-null returns null on analytics; the
  existing isPushdownDisabled branch already handles the Calcite-no-
  pushdown case. OR analytics into that branch.
- testStatsPercentileByNullValue + NonNullBucket — percentile of an
  all-null / empty group returns null on analytics. Same OR pattern.

Non-deterministic head ordering:
- testStatsWithLimit Q1+Q2 — head 5 over a 6-bucket result picks a
  different null-balance row than the legacy / Calcite-DSL path; same
  effect cascades into `head 2 from 1`. Reuse the size-only assertion
  branch already present for the no-pushdown case.

TDigest interpolation divergence (genuine impl difference):
- testStatsPercentileWithNull — DataFusion TDigest interpolates p50 to
  35413 vs OpenSearch's 39225; both within compression-bound error.
  Per-engine expected value.
- testStatsPercentileBySpan — same TDigest diff on the age=30 bucket
  (33194 vs 39225). Coordinated with opensearch-project/OpenSearch#21584
  commit 5 which fixes the upstream planner-side type mismatch so the
  query reaches the data assertion at all.

Semantic divergence pending team decision:
- testDisableLegacyPreferred — under PPL_SYNTAX_LEGACY_PREFERRED=false,
  V2 / Calcite-DSL drop the null-age bucket (5 rows) while DataFusion
  keeps it (6 rows). Skipped on the analytics path via
  Assume.assumeFalse until the team decides which behaviour is the
  intended new-syntax semantics.

Out of scope for this PR (intentionally left failing on the analytics
path so the gap is visible in CI):
- testStatsBySpanTimeWithNullBucket — span(@timestamp, 12h) multi-unit
  time interval unsupported in current SpanAdapter (only interval=1 is
  rewritten to DATE_TRUNC; multi-unit needs DataFusion's date_bin).

Signed-off-by: Kai Huang <ahkcs@amazon.com>

Author: ahkcs

integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java

@@ -309,7 +309,10 @@ public void testStatsWithLimit() throws IOException {
     verifySchema(response, schema("a", null, "double"), schema("age", null, "int"));
     // If push down disabled, the final results will no longer be stable. In DSL, the order is
     // guaranteed because we always sort by bucket field, while we don't add sort in the plan.
-    if (!isPushdownDisabled()) {
+    // The analytics-engine backend (DataFusion) is also non-deterministic — its hash-bucket
+    // order picks a different null-balance row (e.g. (null,36) instead of (null,null)) for
+    // `head 5`.
+    if (!isPushdownDisabled() && !isAnalyticsParquetIndicesEnabled()) {
       verifyDataRows(
           response,
           rows(null, null),
@@ -327,7 +330,8 @@ public void testStatsWithLimit() throws IOException {
                 "source=%s | stats avg(balance) as a by age | head 5 | head 2 from 1",
                 TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifySchema(response, schema("a", null, "double"), schema("age", null, "int"));
-    if (!isPushdownDisabled()) {
+    // Same non-deterministic head-window issue as the preceding `head 5` block.
+    if (!isPushdownDisabled() && !isAnalyticsParquetIndicesEnabled()) {
       verifyDataRows(response, rows(32838D, 28), rows(39225D, 32));
     } else {
       assert ((Integer) response.get("size") == 2);
@@ -508,7 +512,9 @@ public void testSumWithNull() throws IOException {
     // TODO: Fix -- temporary workaround for the pushdown issue:
     //  The current pushdown implementation will return 0 for sum when getting null values as input.
     //  Returning null should be the expected behavior.
-    Integer expectedValue = isPushdownDisabled() ? null : 0;
+    // The analytics-engine backend (DataFusion) follows the SQL spec like Calcite-no-pushdown —
+    // SUM of all-null is null, not 0.
+    Integer expectedValue = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0;
     verifyDataRows(response, rows(expectedValue));
   }
 
@@ -635,7 +641,11 @@ public void testStatsPercentileWithNull() throws IOException {
             String.format(
                 "source=%s | stats percentile(balance, 50)", TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifySchema(response, schema("percentile(balance, 50)", null, "bigint"));
-    verifyDataRows(response, rows(39225));
+    // DataFusion's TDigest interpolation produces a different median value for the same
+    // input than OpenSearch's TDigest implementation. Both are valid approximations within
+    // TDigest's compression-bound error. Until DataFusion's UDAF is replaced with one
+    // matching OpenSearch's TDigest (or vice versa), record the per-engine values.
+    verifyDataRows(response, rows(isAnalyticsParquetIndicesEnabled() ? 35413 : 39225));
   }
 
   @Test
@@ -674,14 +684,18 @@ public void testStatsPercentileByNullValue() throws IOException {
                 "source=%s | stats percentile(balance, 50) as p50 by age",
                 TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifySchema(response, schema("p50", null, "bigint"), schema("age", null, "int"));
+    // DataFusion follows SQL spec like Calcite-no-pushdown — percentile of an all-null
+    // group is null, not 0 (the latter is a legacy DSL pushdown convention).
+    Integer emptyGroupPercentile =
+        (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0;
     verifyDataRows(
         response,
-        rows(isPushdownDisabled() ? null : 0, null),
+        rows(emptyGroupPercentile, null),
         rows(32838, 28),
         rows(39225, 32),
         rows(4180, 33),
         rows(48086, 34),
-        rows(isPushdownDisabled() ? null : 0, 36));
+        rows(emptyGroupPercentile, 36));
   }
 
   @Test
@@ -692,13 +706,15 @@ public void testStatsPercentileByNullValueNonNullBucket() throws IOException {
                 "source=%s | stats bucket_nullable=false percentile(balance, 50) as p50 by age",
                 TEST_INDEX_BANK_WITH_NULL_VALUES));
     verifySchema(response, schema("p50", null, "bigint"), schema("age", null, "int"));
+    Integer emptyGroupPercentile =
+        (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0;
     verifyDataRows(
         response,
         rows(32838, 28),
         rows(39225, 32),
         rows(4180, 33),
         rows(48086, 34),
-        rows(isPushdownDisabled() ? null : 0, 36));
+        rows(emptyGroupPercentile, 36));
   }
 
   @Test
@@ -709,7 +725,14 @@ public void testStatsPercentileBySpan() throws IOException {
                 "source=%s | stats percentile(balance, 50) as p50 by span(age, 10) as age_bucket",
                 TEST_INDEX_BANK));
     verifySchema(response, schema("p50", null, "bigint"), schema("age_bucket", null, "int"));
-    verifyDataRows(response, rows(32838, 20), rows(39225, 30));
+    // Same per-engine TDigest interpolation divergence as testStatsPercentileWithNull —
+    // the age=30 bucket's p50 differs between OpenSearch's TDigest (39225) and DataFusion's
+    // (33194). The age=20 bucket happens to coincide.
+    if (isAnalyticsParquetIndicesEnabled()) {
+      verifyDataRows(response, rows(32838, 20), rows(33194, 30));
+    } else {
+      verifyDataRows(response, rows(32838, 20), rows(39225, 30));
+    }
   }
 
   @Test
@@ -729,6 +752,15 @@ public void testDisableLegacyPreferred() throws IOException {
             throw new RuntimeException(e);
           }
           verifySchema(response, schema("a", null, "double"), schema("age", null, "int"));
+          // Skip on the analytics-engine backend: PPL_SYNTAX_LEGACY_PREFERRED=false is
+          // expected to give the same shape across backends, but v2 / Calcite-DSL drop
+          // the null-age bucket (5 rows) while DataFusion keeps it (6 rows). Deciding
+          // which is correct is a semantic question for the team; until that's resolved,
+          // exercising this test on the analytics path doesn't tell us anything useful.
+          org.junit.Assume.assumeFalse(
+              "PPL_SYNTAX_LEGACY_PREFERRED=false null-bucket semantics not yet aligned"
+                  + " between V2 / Calcite-DSL and analytics-engine backends",
+              isAnalyticsParquetIndicesEnabled());
           verifyDataRows(
               response,
               rows(32838D, 28),