Merge upstream/main into feature/mustang-ppl-integration

Brings the branch up to current upstream/main (54 commits since the last
catch-up at #5397, divergence point 513e1b2). Preserves both parents
so commit lineage from main is retained on the feature branch.

### Main commits absorbed (54 since divergence)

Highlights:
- #5419 Register LENGTH/REGEXP_REPLACE/DATE_TRUNC in unified function spec
- #5408 Normalize datetime types for unified query API
- #5414 Gradle wrapper 9.4.1 bump + exclude @ignore classes
- #5399 [BugFix] Scope SQL cursor continuation to original query indices under FGAC
- #5394 [Feature] Support SQL Vector Search
- #5361 Version bump to OpenSearch 3.7 (Jackson 2 → 3 parser API)
- #5360 Define unified SQL language spec with composable extensions
- #5240 [FEATURE] Union command in PPL
- #5266 Initial implementation of report-builder interface
- #5278 isnotnull condition support
- #3922, #4659, #4800, #5099, #5169, #5172, #5175, #5185, #5347, #5370
  and other bugfixes
- 34 others (bugfixes, doc updates, infra)

### Conflict resolutions

Seven content conflicts resolved during the merge. Resolution kept the
feature branch's analytics-engine-path semantics where main's changes
would have regressed them.

- api/.../UnifiedQueryContext.java
  Blank-line-only conflict in Builder.build(); took main's tighter
  formatting (no extra blank).

- core/.../executor/QueryService.java
  Main reformatted executeWithCalcite without the
  CalciteClassLoaderHelper.withCalciteClassLoader() wrapper. Kept the
  feature-branch wrapping — required for analytics-engine classloader
  isolation — and the matching import. Same shape applies to
  explainWithCalcite which git auto-merged correctly.

- integ-test/build.gradle
  Both sides added @Ignore-class exclusions to work around the Gradle
  9.4.1 TestEventReporterAsListener ClassCastException. Took the
  feature-branch's detailed root-cause comment, kept ASCII ordering of
  the JSONRequestIT/JoinIT and SQLFunctionsIT/ShowIT/SourceFieldIT
  entries.

- integ-test/.../CalciteEvalCommandIT.java
  Feature branch wraps test-data PUTs in
  if (!TestUtils.isIndexExist(...)) for idempotency on the
  helper-managed-index analytics-engine compatibility run; main has the
  unguarded PUTs. Kept feature's idempotency guards for both test_eval
  and test_eval_agent.

- legacy/.../RestSqlAction.java
  Feature branch added delegateToV2Engine(...) (extracted from the
  analytics-engine routing path) and ALSO has handleException /
  getRestStatus / getRawErrorCode; main added the latter three in the
  same position. Kept feature's delegateToV2Engine, kept one copy of
  the helpers, removed the duplicate set that git produced.

- plugin/.../SQLPlugin.java
  Both sides expanded the executor import block. Kept the union:
  ExecutionEngine + ExecutionEngine.ExplainResponse + QueryType (all
  three are referenced in this file).

- plugin/.../transport/TransportPPLQueryAction.java
  Main now passes engine extensions to OpenSearchPluginModule via
  extensionsHolder.engines(). Feature creates a local pluginSettings
  for the unified-query handler. Combined both — kept the
  extensions-aware module construction *and* the pluginSettings /
  pluginSettingsRef wiring.

EngineExtensionsHolder is a new file from main (#5298) preserved as-is.

### Verification

- ./gradlew :api:compileJava :core:compileJava :legacy:compileJava
  :opensearch-sql-plugin:compileJava :integ-test:compileTestJava
  passes locally

Signed-off-by: Kai Huang <ahkcs@amazon.com>

Author: ahkcs

api/src/main/java/org/opensearch/sql/api/UnifiedQueryContext.java

@@ -234,7 +234,6 @@ public UnifiedQueryContext build() {
             case SQL -> UnifiedSqlSpec.extended();
             case PPL -> UnifiedPplSpec.create();
           };
-
       Settings settings = buildSettings();
       CalcitePlanContext planContext =
           CalcitePlanContext.create(

integ-test/build.gradle

@@ -601,8 +601,8 @@ integTest {
     // even though the tests would have been skipped anyway. The bridge is registered by Gradle's
     // own AbstractTestTask (we can't bypass it from user code), so the only available remedy is
     // to keep these classes off the test runner's input set. Net behaviour for CI: still
-    // skipped, just at the build layer instead of inside JUnit. Remove once Gradle ships a fix.
-    // OrderIT is already excluded above.
+    // skipped, just at the build layer instead of inside JUnit. Remove once Gradle ships a fix
+    // (not fixed as of 9.5.0). OrderIT is already excluded above.
     exclude 'org/opensearch/sql/calcite/remote/CalciteInformationSchemaCommandIT.class'
     exclude 'org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.class'
     exclude 'org/opensearch/sql/calcite/remote/CalcitePrometheusDataSourceCommandsIT.class'

plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java

@@ -49,6 +49,7 @@
 import org.opensearch.jobscheduler.spi.ScheduledJobParser;
 import org.opensearch.jobscheduler.spi.ScheduledJobRunner;
 import org.opensearch.plugins.ActionPlugin;
+import org.opensearch.plugins.ExtensiblePlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.ScriptPlugin;
 import org.opensearch.plugins.SystemIndexPlugin;
@@ -92,6 +93,7 @@
 import org.opensearch.sql.directquery.transport.model.ExecuteDirectQueryActionResponse;
 import org.opensearch.sql.directquery.transport.model.ReadDirectQueryResourcesActionResponse;
 import org.opensearch.sql.directquery.transport.model.WriteDirectQueryResourcesActionResponse;
+import org.opensearch.sql.executor.ExecutionEngine;
 import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse;
 import org.opensearch.sql.executor.QueryType;
 import org.opensearch.sql.legacy.esdomain.LocalClusterState;
@@ -102,6 +104,7 @@
 import org.opensearch.sql.opensearch.setting.OpenSearchSettings;
 import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory;
 import org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine;
+import org.opensearch.sql.plugin.config.EngineExtensionsHolder;
 import org.opensearch.sql.plugin.config.OpenSearchPluginModule;
 import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder;
 import org.opensearch.sql.plugin.rest.RestPPLGrammarAction;
@@ -140,10 +143,15 @@
 import org.opensearch.watcher.ResourceWatcherService;
 
 public class SQLPlugin extends Plugin
-    implements ActionPlugin, ScriptPlugin, SystemIndexPlugin, JobSchedulerExtension {
+    implements ActionPlugin,
+        ScriptPlugin,
+        SystemIndexPlugin,
+        JobSchedulerExtension,
+        ExtensiblePlugin {
 
   private static final Logger LOGGER = LogManager.getLogger(SQLPlugin.class);
 
+  private List<ExecutionEngine> executionEngineExtensions = List.of();
   private ClusterService clusterService;
 
   /** Settings should be inited when bootstrap the plugin. */
@@ -162,6 +170,18 @@ public String description() {
     return "Use sql to query OpenSearch.";
   }
 
+  @Override
+  public void loadExtensions(ExtensionLoader loader) {
+    List<ExecutionEngine> loaded = loader.loadExtensions(ExecutionEngine.class);
+    this.executionEngineExtensions = loaded != null ? List.copyOf(loaded) : List.of();
+    if (!executionEngineExtensions.isEmpty()) {
+      LOGGER.info(
+          "Loaded {} execution engine extension(s): {}",
+          executionEngineExtensions.size(),
+          executionEngineExtensions.stream().map(e -> e.getClass().getSimpleName()).toList());
+    }
+  }
+
   @Override
   public List<RestHandler> getRestHandlers(
       Settings settings,
@@ -348,7 +368,7 @@ public Collection<Object> createComponents(
     LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings);
     LocalClusterState.state().setClient(client);
     ModulesBuilder modules = new ModulesBuilder();
-    modules.add(new OpenSearchPluginModule());
+    modules.add(new OpenSearchPluginModule(executionEngineExtensions));
     modules.add(
         b -> {
           b.bind(NodeClient.class).toInstance((NodeClient) client);
@@ -383,12 +403,15 @@ public Collection<Object> createComponents(
     ScheduledAsyncQueryJobRunner.getJobRunnerInstance()
         .loadJobResource(client, clusterService, threadPool, asyncQueryExecutorService);
 
+    EngineExtensionsHolder extensionsHolder = new EngineExtensionsHolder(executionEngineExtensions);
+
     return ImmutableList.of(
         dataSourceService,
         asyncQueryExecutorService,
         clusterManagerEventListener,
         pluginSettings,
-        directQueryExecutorService);
+        directQueryExecutorService,
+        extensionsHolder);
   }
 
   @Override

plugin/src/main/java/org/opensearch/sql/plugin/config/EngineExtensionsHolder.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.plugin.config;
+
+import java.util.List;
+import org.opensearch.sql.executor.ExecutionEngine;
+
+/**
+ * Holds execution engine engines loaded via SPI. Returned from {@code SQLPlugin.createComponents()}
+ * so that OpenSearch's Guice injector can inject it into transport actions like {@code
+ * TransportPPLQueryAction}.
+ */
+public record EngineExtensionsHolder(List<ExecutionEngine> engines) {
+  public EngineExtensionsHolder(List<ExecutionEngine> engines) {
+    this.engines = engines != null ? List.copyOf(engines) : List.of();
+  }
+}

plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java

@@ -36,6 +36,7 @@
 import org.opensearch.sql.monitor.profile.QueryProfiling;
 import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager;
 import org.opensearch.sql.opensearch.setting.OpenSearchSettings;
+import org.opensearch.sql.plugin.config.EngineExtensionsHolder;
 import org.opensearch.sql.plugin.config.OpenSearchPluginModule;
 import org.opensearch.sql.plugin.rest.AnalyticsExecutorHolder;
 import org.opensearch.sql.plugin.rest.RestUnifiedQueryAction;
@@ -76,13 +77,14 @@ public TransportPPLQueryAction(
       NodeClient client,
       ClusterService clusterService,
       DataSourceServiceImpl dataSourceService,
-      org.opensearch.common.settings.Settings clusterSettings) {
+      org.opensearch.common.settings.Settings clusterSettings,
+      EngineExtensionsHolder extensionsHolder) {
     super(PPLQueryAction.NAME, transportService, actionFilters, TransportPPLQueryRequest::new);
     this.clientRef = client;
     this.clusterServiceRef = clusterService;
 
     ModulesBuilder modules = new ModulesBuilder();
-    modules.add(new OpenSearchPluginModule());
+    modules.add(new OpenSearchPluginModule(extensionsHolder.engines()));
     org.opensearch.sql.common.setting.Settings pluginSettings =
         new OpenSearchSettings(clusterService.getClusterSettings());
     this.pluginSettingsRef = pluginSettings;