Merge upstream/feature/mustang-ppl-integration into PR catch-up branch

The feature branch advanced 9 commits since this PR was opened. Bring
the catch-up branch up to that tip so the PR is mergeable into
feature/mustang-ppl-integration.

Resolutions:

integ-test/build.gradle: took feature. Both sides added the same
@ignore exclusion block; feature has alphabetical ordering and a more
detailed comment explaining the Gradle 9.4.1 TestEventReporterAsListener
cast bug.

integ-test/.../CalciteEvalCommandIT.java: composed both sides. Took
feature's helper-managed test_eval provisioning (createIndexByRestClient
+ isIndexExist guard, from #5407) so analytics-engine compatibility runs
get a parquet-backed index. Added back PR HEAD's test_eval_agent setup
(needed by the dotted-path eval tests for #5351) wrapped in its own
isIndexExist guard for the same parquet-aware idempotency.

plugin/.../TransportPPLQueryAction.java: took feature. PR #5403 made
analytics-engine an optional dependency by moving QueryPlanExecutor
from a required constructor parameter to an @Inject(optional=true)
setter. Feature's design supersedes our prior wiring.

plugin/.../SQLPlugin.java: took feature. The same #5403 simplification
removed loadExtensions/EngineExtensionsHolder/executionEngineExtensions
plumbing (no longer needed once analytics-engine is optionally bound).
Feature retains the createSqlAnalyticsRouter method this PR introduced.

plugin/.../config/EngineExtensionsHolder.java: deleted. Unreferenced
after taking feature's SQLPlugin/TransportPPLQueryAction; not present
on feature branch.

Build: :api, :core, :opensearch-sql-plugin, :legacy compile +
:integ-test compileTestJava all pass; unit tests pass; spotless clean.

Signed-off-by: Kai Huang <ahkcs@amazon.com>

Author: ahkcs

api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java

@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.api;
 
+import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
 import static org.opensearch.sql.common.setting.Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT;
 import static org.opensearch.sql.common.setting.Settings.Key.PPL_SUBSEARCH_MAXOUT;
 import static org.opensearch.sql.common.setting.Settings.Key.QUERY_SIZE_LIMIT;
@@ -119,13 +120,21 @@ public static class Builder {
     /**
      * Setting values with defaults from SysLimit.DEFAULT. Only includes planning-required settings
      * to avoid coupling with OpenSearchSettings.
+     *
+     * <p>{@link Settings.Key#CALCITE_ENGINE_ENABLED} defaults to {@code true} here because the
+     * unified query path is by definition Calcite-based — every query reaching this context flows
+     * through Calcite's planner, never the v2 engine. The PPL {@link
+     * org.opensearch.sql.api.parser.PPLQueryParser} reuses the v2 {@code AstBuilder}, which gates
+     * Calcite-only commands (e.g. {@code visitTableCommand}) on this setting; without the default,
+     * those commands fail at parse time even when the cluster setting is true.
      */
     private final Map<Settings.Key, Object> settings =
         new HashMap<Settings.Key, Object>(
             Map.of(
                 QUERY_SIZE_LIMIT, SysLimit.DEFAULT.querySizeLimit(),
                 PPL_SUBSEARCH_MAXOUT, SysLimit.DEFAULT.subsearchLimit(),
-                PPL_JOIN_SUBSEARCH_MAXOUT, SysLimit.DEFAULT.joinSubsearchLimit()));
+                PPL_JOIN_SUBSEARCH_MAXOUT, SysLimit.DEFAULT.joinSubsearchLimit(),
+                CALCITE_ENGINE_ENABLED, true));
 
     /**
      * Sets the query language frontend to be used.

integ-test/build.gradle

@@ -557,10 +557,15 @@ integTest {
     // Exclude this IT, because they executed in another task (:integTestWithSecurity)
     exclude 'org/opensearch/sql/security/**'
 
-    // Workaround for Gradle 9.4.1 bug: TestEventReporterAsListener crashes with ClassCastException
-    // when encountering class-level @Ignore annotations. These classes were already skipped by JUnit;
-    // this moves the skip to the Gradle layer to avoid the buggy bridge.
-    // Remove once Gradle ships a fix (not fixed as of 9.5.0).
+    // Workaround for Gradle 9.4.1 ClassCastException in TestEventReporterAsListener.started
+    // (line 58) — the bridge casts a parent test descriptor's reporter to
+    // GroupTestEventReporterInternal but a class-level @Ignore produces a non-composite parent
+    // descriptor with a leaf reporter, so the cast fails and aborts the entire integTest task
+    // even though the tests would have been skipped anyway. The bridge is registered by Gradle's
+    // own AbstractTestTask (we can't bypass it from user code), so the only available remedy is
+    // to keep these classes off the test runner's input set. Net behaviour for CI: still
+    // skipped, just at the build layer instead of inside JUnit. Remove once Gradle ships a fix.
+    // OrderIT is already excluded above.
     exclude 'org/opensearch/sql/calcite/remote/CalciteInformationSchemaCommandIT.class'
     exclude 'org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.class'
     exclude 'org/opensearch/sql/calcite/remote/CalcitePrometheusDataSourceCommandsIT.class'
@@ -570,18 +575,18 @@ integTest {
     exclude 'org/opensearch/sql/legacy/DateFunctionsIT.class'
     exclude 'org/opensearch/sql/legacy/HashJoinIT.class'
     exclude 'org/opensearch/sql/legacy/HavingIT.class'
-    exclude 'org/opensearch/sql/legacy/JoinIT.class'
     exclude 'org/opensearch/sql/legacy/JSONRequestIT.class'
+    exclude 'org/opensearch/sql/legacy/JoinIT.class'
     exclude 'org/opensearch/sql/legacy/MathFunctionsIT.class'
     exclude 'org/opensearch/sql/legacy/MetricsIT.class'
     exclude 'org/opensearch/sql/legacy/MultiQueryIT.class'
     exclude 'org/opensearch/sql/legacy/NestedFieldQueryIT.class'
     exclude 'org/opensearch/sql/legacy/PreparedStatementIT.class'
     exclude 'org/opensearch/sql/legacy/QueryFunctionsIT.class'
     exclude 'org/opensearch/sql/legacy/QueryIT.class'
+    exclude 'org/opensearch/sql/legacy/SQLFunctionsIT.class'
     exclude 'org/opensearch/sql/legacy/ShowIT.class'
     exclude 'org/opensearch/sql/legacy/SourceFieldIT.class'
-    exclude 'org/opensearch/sql/legacy/SQLFunctionsIT.class'
     exclude 'org/opensearch/sql/legacy/SubqueryIT.class'
     exclude 'org/opensearch/sql/ppl/JsonFunctionsIT.class'
     exclude 'org/opensearch/sql/sql/ExpressionIT.class'

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteEvalCommandIT.java

@@ -17,6 +17,7 @@
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
 import org.opensearch.client.Request;
+import org.opensearch.sql.legacy.TestUtils;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
 
 public class CalciteEvalCommandIT extends PPLIntegTestCase {
@@ -29,38 +30,63 @@ public void init() throws Exception {
     loadIndex(Index.BANK);
     loadIndex(Index.TELEMETRY);
 
-    // Create test data for string concatenation
-    Request request1 = new Request("PUT", "/test_eval/_doc/1?refresh=true");
-    request1.setJsonEntity("{\"name\": \"Alice\", \"age\": 25, \"title\": \"Engineer\"}");
-    client().performRequest(request1);
-
-    Request request2 = new Request("PUT", "/test_eval/_doc/2?refresh=true");
-    request2.setJsonEntity("{\"name\": \"Bob\", \"age\": 30, \"title\": \"Manager\"}");
-    client().performRequest(request2);
-
-    Request request3 = new Request("PUT", "/test_eval/_doc/3?refresh=true");
-    request3.setJsonEntity("{\"name\": \"Charlie\", \"age\": null, \"title\": \"Analyst\"}");
-    client().performRequest(request3);
+    // Pre-create test_eval through the helper so the analytics-engine compatibility run
+    // (tests.analytics.parquet_indices=true) provisions it as a parquet-backed composite
+    // index. Plain auto-mapping via the doc PUTs would create a Lucene-backed index, which
+    // the analytics-engine planner cannot scan ("No backend can scan all requested fields").
+    // Explicit mapping pins types so both v2 (verifySchema "string"/"bigint") and analytics
+    // paths see the same shape regardless of dynamic-mapping behavior on the parquet engine.
+    // Guarded by isIndexExist for idempotency — init() runs before each @Test method.
+    if (!TestUtils.isIndexExist(client(), "test_eval")) {
+      String testEvalMapping =
+          "{\"mappings\":{\"properties\":{"
+              + "\"name\":{\"type\":\"keyword\"},"
+              + "\"age\":{\"type\":\"long\"},"
+              + "\"title\":{\"type\":\"keyword\"}}}}";
+      TestUtils.createIndexByRestClient(client(), "test_eval", testEvalMapping);
+
+      // Create test data for string concatenation
+      Request request1 = new Request("PUT", "/test_eval/_doc/1?refresh=true");
+      request1.setJsonEntity("{\"name\": \"Alice\", \"age\": 25, \"title\": \"Engineer\"}");
+      client().performRequest(request1);
+
+      Request request2 = new Request("PUT", "/test_eval/_doc/2?refresh=true");
+      request2.setJsonEntity("{\"name\": \"Bob\", \"age\": 30, \"title\": \"Manager\"}");
+      client().performRequest(request2);
+
+      Request request3 = new Request("PUT", "/test_eval/_doc/3?refresh=true");
+      request3.setJsonEntity("{\"name\": \"Charlie\", \"age\": null, \"title\": \"Analyst\"}");
+      client().performRequest(request3);
+    }
 
     // Index with a struct field `agent` to reproduce the reviewer's case from PR #5351:
     //   source=<idx with agent struct> | fields agent | eval agent.name = "test"
     // Rely on dynamic mapping — OpenSearch infers `agent` as an object with string children
     // from the document contents. Using dynamic mapping keeps the init idempotent across
     // repeated `@Before` invocations in the preserved cluster.
-    Request agentDoc1 = new Request("PUT", "/test_eval_agent/_doc/1?refresh=true");
-    agentDoc1.setJsonEntity(
-        "{\"agent\": {\"name\": \"winlogbeat\", \"version\": \"7.0\"}, \"message\": \"hello\"}");
-    client().performRequest(agentDoc1);
-
-    Request agentDoc2 = new Request("PUT", "/test_eval_agent/_doc/2?refresh=true");
-    agentDoc2.setJsonEntity(
-        "{\"agent\": {\"name\": \"filebeat\", \"version\": \"8.1\"}, \"message\": \"world\"}");
-    client().performRequest(agentDoc2);
+    if (!TestUtils.isIndexExist(client(), "test_eval_agent")) {
+      Request agentDoc1 = new Request("PUT", "/test_eval_agent/_doc/1?refresh=true");
+      agentDoc1.setJsonEntity(
+          "{\"agent\": {\"name\": \"winlogbeat\", \"version\": \"7.0\"}, \"message\": \"hello\"}");
+      client().performRequest(agentDoc1);
+
+      Request agentDoc2 = new Request("PUT", "/test_eval_agent/_doc/2?refresh=true");
+      agentDoc2.setJsonEntity(
+          "{\"agent\": {\"name\": \"filebeat\", \"version\": \"8.1\"}, \"message\": \"world\"}");
+      client().performRequest(agentDoc2);
+    }
   }
 
   @Test
   public void testEvalStringConcatenation() throws IOException {
-    JSONObject result = executeQuery("source=test_eval | eval greeting = 'Hello ' + name");
+    // Pin the projection so column order is deterministic across execution paths — the
+    // analytics-engine route reads parquet schema in storage order, which can differ from the
+    // v2 / Lucene path's _source-iteration order. Adding an explicit | fields makes the test
+    // a strict assertion on the eval expression rather than a coincidence of projection order.
+    JSONObject result =
+        executeQuery(
+            "source=test_eval | eval greeting = 'Hello ' + name | fields name, title, age,"
+                + " greeting");
     verifySchema(
         result,
         schema("name", "string"),

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java

@@ -12,6 +12,7 @@
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
 import org.opensearch.client.Request;
+import org.opensearch.sql.legacy.TestUtils;
 import org.opensearch.sql.ppl.PPLIntegTestCase;
 
 public class CalciteFieldFormatCommandIT extends PPLIntegTestCase {
@@ -23,26 +24,48 @@ public void init() throws Exception {
 
     loadIndex(Index.BANK);
 
-    // Create test data for string concatenation
-    Request request1 = new Request("PUT", "/test_eval/_doc/1?refresh=true");
-    request1.setJsonEntity("{\"name\": \"Alice\", \"age\": 25, \"title\": \"Engineer\"}");
-    client().performRequest(request1);
+    // Pre-create test_eval through the helper so the analytics-engine compatibility run
+    // (tests.analytics.parquet_indices=true) provisions it as a parquet-backed composite
+    // index. Plain auto-mapping via the doc PUTs would create a Lucene-backed index, which
+    // the analytics-engine planner cannot scan ("No backend can scan all requested fields").
+    // Explicit mapping pins types so both v2 (verifySchema "string"/"bigint") and analytics
+    // paths see the same shape regardless of dynamic-mapping behavior on the parquet engine.
+    // Guarded by isIndexExist for idempotency — init() runs before each @Test method.
+    if (!TestUtils.isIndexExist(client(), "test_eval")) {
+      String testEvalMapping =
+          "{\"mappings\":{\"properties\":{"
+              + "\"name\":{\"type\":\"keyword\"},"
+              + "\"age\":{\"type\":\"long\"},"
+              + "\"title\":{\"type\":\"keyword\"}}}}";
+      TestUtils.createIndexByRestClient(client(), "test_eval", testEvalMapping);
 
-    Request request2 = new Request("PUT", "/test_eval/_doc/2?refresh=true");
-    request2.setJsonEntity("{\"name\": \"Bob\", \"age\": 30, \"title\": \"Manager\"}");
-    client().performRequest(request2);
+      // Create test data for string concatenation
+      Request request1 = new Request("PUT", "/test_eval/_doc/1?refresh=true");
+      request1.setJsonEntity("{\"name\": \"Alice\", \"age\": 25, \"title\": \"Engineer\"}");
+      client().performRequest(request1);
 
-    Request request3 = new Request("PUT", "/test_eval/_doc/3?refresh=true");
-    request3.setJsonEntity("{\"name\": \"Charlie\", \"age\": null, \"title\": \"Analyst\"}");
-    client().performRequest(request3);
+      Request request2 = new Request("PUT", "/test_eval/_doc/2?refresh=true");
+      request2.setJsonEntity("{\"name\": \"Bob\", \"age\": 30, \"title\": \"Manager\"}");
+      client().performRequest(request2);
+
+      Request request3 = new Request("PUT", "/test_eval/_doc/3?refresh=true");
+      request3.setJsonEntity("{\"name\": \"Charlie\", \"age\": null, \"title\": \"Analyst\"}");
+      client().performRequest(request3);
+    }
   }
 
   @Test
   public void testFieldFormatStringConcatenation() throws IOException {
+    // Pin the projection so column order is deterministic across execution paths — the
+    // analytics-engine route reads parquet schema in storage order, which can differ from the
+    // v2 / Lucene path's _source-iteration order. Adding an explicit | fields makes the test
+    // a strict assertion on the fieldformat expression rather than a coincidence of projection
+    // order.
     JSONObject result =
         executeQuery(
             StringEscapeUtils.escapeJson(
-                "source=test_eval | fieldformat greeting = 'Hello ' + name"));
+                "source=test_eval | fieldformat greeting = 'Hello ' + name | fields name, title,"
+                    + " age, greeting"));
     verifySchema(
         result,
         schema("name", "string"),