Make analytics-engine an optional dependency

sql plugin previously declared analytics-engine as a hard dependency:

    extendedPlugins = ['opensearch-job-scheduler', 'analytics-engine']

Installing opensearch-sql on a distribution that doesn't ship
analytics-engine failed with:

    Missing plugin [analytics-engine], dependency of [opensearch-sql]

Marking the dep ;optional=true alone is not enough — TransportPPLQueryAction
Guice-injects QueryPlanExecutor on its constructor, and Guice's OpenSearch
fork rejects a required constructor parameter whose binding is missing at
injector-build time ("constructors cannot be optional").

Move QueryPlanExecutor from a required constructor parameter to an
@Inject(optional=true) setter. Guice invokes the setter only when a binding
for QueryPlanExecutor<RelNode, Iterable<Object[]>> exists — i.e. when
analytics-engine's createGuiceModules has run and bound DefaultPlanExecutor.
Absent analytics-engine, the setter is silently skipped,
unifiedQueryHandler stays null, and all PPL queries route to the v2
Calcite-to-OpenSearch path already in the sql plugin.

Drop the bundlePlugin exclude list. OpenSearch's jar-hell check skips the
extended-plugin cross-check when the dep is marked optional
(PluginsService.java:763), so sql can bundle every jar it needs to run
self-sufficiently. When analytics-engine is installed, parent-first
classloader delegation still lets analytics-engine's copies win for any
shared class; sql's bundled copies sit idle.

Promote analytics-framework.jar from compileOnly to api so
QueryPlanExecutor is reachable from sql's own classloader when the plugin
is absent. analytics-engine.jar stays compileOnly (required only for
OpenSearchSchemaBuilder, which lives in the engine plugin and is reached
only through RestUnifiedQueryAction — a class that never loads when the
setter is skipped).

Validated on a live 2-node cluster in both configurations:

- With analytics-engine installed: legacy and analytics PPL both return
  expected rows; routing to the analytics path still fires for
  parquet_-prefixed indices.
- Without analytics-engine (only opensearch-job-scheduler + opensearch-sql
  installed): cluster starts cleanly, PPL and SQL queries against Lucene
  indices return expected rows, parquet_-prefixed lookups return a clean
  IndexNotFoundException instead of a NullPointerException or
  NoClassDefFoundError.

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>

Author: bowenlan-amzn

plugin/build.gradle

@@ -55,57 +55,11 @@ opensearchplugin {
     name 'opensearch-sql'
     description 'OpenSearch SQL'
     classname 'org.opensearch.sql.plugin.SQLPlugin'
-    extendedPlugins = ['opensearch-job-scheduler', 'analytics-engine']
+    extendedPlugins = ['opensearch-job-scheduler', 'analytics-engine;optional=true']
     licenseFile rootProject.file("LICENSE.txt")
     noticeFile rootProject.file("NOTICE")
 }
 
-// Exclude jars provided by the analytics-engine plugin (shared via extendedPlugins classloader).
-// MUST match what's actually in the analytics-engine ZIP to avoid both jar hell (when analytics-engine
-// ships it) and ClassNotFoundException at runtime (when it doesn't). Last verified against
-// analytics-engine-3.7.0-SNAPSHOT — keep this list aligned when bumping versions.
-bundlePlugin {
-    exclude 'calcite-core-*.jar'
-    exclude 'calcite-linq4j-*.jar'
-    exclude 'avatica-core-*.jar'
-    exclude 'guava-*.jar'
-    exclude 'failureaccess-*.jar'
-    exclude 'slf4j-api-*.jar'
-    exclude 'commons-codec-*.jar'
-    exclude 'commons-compiler-*.jar'
-    exclude 'commons-io-*.jar'
-    exclude 'commons-lang3-*.jar'
-    exclude 'janino-*.jar'
-    exclude 'joou-java-6-*.jar'
-    exclude 'json-path-*.jar'
-    exclude 'jackson-annotations-*.jar'
-    exclude 'jackson-databind-*.jar'
-    exclude 'httpcore5-5*.jar'
-    // TODO: Remove the three httpcore5/httpclient5 exclusions below — and ideally this entire
-    // bundlePlugin exclusion block — once analytics-engine becomes an optional dependency via the
-    // AnalyticsFrontEndExtension SPI (opensearch-project/OpenSearch#21449). The shared-classloader
-    // jar deduplication that requires this hand-maintained list goes away with the SPI.
-    exclude 'httpcore5-h2-*.jar'
-    exclude 'httpcore5-reactive-*.jar'
-    exclude 'httpclient5-*.jar'
-    exclude 'commons-text-*.jar'
-    // Calcite transitive deps now bundled in analytics-engine 3.7 — exclude from sql to avoid jar hell.
-    exclude 'commons-math3-*.jar'
-    exclude 'commons-dbcp2-*.jar'
-    exclude 'commons-pool2-*.jar'
-    exclude 'uzaygezen-core-*.jar'
-    exclude 'sketches-core-*.jar'
-    exclude 'memory-0*.jar'
-    exclude 'jts-io-common-*.jar'
-    exclude 'proj4j-*.jar'
-    exclude 'json-smart-*.jar'
-    exclude 'accessors-smart-*.jar'
-    exclude 'asm-9*.jar'
-    // jsr305 is also bundled by arrow-flight-rpc (an extendedPlugins parent of analytics-engine,
-    // which we extend); both shipping it caused jar hell at plugin install time.
-    exclude 'jsr305-*.jar'
-}
-
 publishing {
     publications {
         pluginZip(MavenPublication) { publication ->
@@ -207,7 +161,9 @@ dependencies {
 
     api project(":ppl")
     api project(':api')
-    compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    // Bundled: analytics-framework interfaces must resolve even when the plugin is absent.
+    api files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    // Not bundled: classes here (e.g. OpenSearchSchemaBuilder) only load when the plugin is installed.
     compileOnly files("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.jar")
     api project(':legacy')
     api project(':opensearch')

plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java

@@ -62,19 +62,23 @@ public class TransportPPLQueryAction
 
   private final Supplier<Boolean> pplEnabled;
 
-  private final RestUnifiedQueryAction unifiedQueryHandler;
+  /** Null when analytics-engine plugin is absent; set via {@link #setQueryPlanExecutor}. */
+  private volatile RestUnifiedQueryAction unifiedQueryHandler;
+
+  private final NodeClient clientRef;
+  private final ClusterService clusterServiceRef;
 
-  /** Constructor of TransportPPLQueryAction. */
   @Inject
   public TransportPPLQueryAction(
       TransportService transportService,
       ActionFilters actionFilters,
       NodeClient client,
       ClusterService clusterService,
       DataSourceServiceImpl dataSourceService,
-      org.opensearch.common.settings.Settings clusterSettings,
-      QueryPlanExecutor<RelNode, Iterable<Object[]>> queryPlanExecutor) {
+      org.opensearch.common.settings.Settings clusterSettings) {
     super(PPLQueryAction.NAME, transportService, actionFilters, TransportPPLQueryRequest::new);
+    this.clientRef = client;
+    this.clusterServiceRef = clusterService;
 
     ModulesBuilder modules = new ModulesBuilder();
     modules.add(new OpenSearchPluginModule());
@@ -86,9 +90,6 @@ public TransportPPLQueryAction(
           b.bind(DataSourceService.class).toInstance(dataSourceService);
         });
     this.injector = Guice.createInjector(modules);
-    AnalyticsExecutorHolder.set(queryPlanExecutor);
-    this.unifiedQueryHandler =
-        new RestUnifiedQueryAction(client, clusterService, queryPlanExecutor);
     this.pplEnabled =
         () ->
             MULTI_ALLOW_EXPLICIT_INDEX.get(clusterSettings)
@@ -98,6 +99,15 @@ public TransportPPLQueryAction(
                         .getSettingValue(Settings.Key.PPL_ENABLED);
   }
 
+  /** Invoked by Guice iff analytics-engine bound {@code QueryPlanExecutor}. */
+  @Inject(optional = true)
+  public void setQueryPlanExecutor(
+      QueryPlanExecutor<RelNode, Iterable<Object[]>> queryPlanExecutor) {
+    AnalyticsExecutorHolder.set(queryPlanExecutor);
+    this.unifiedQueryHandler =
+        new RestUnifiedQueryAction(clientRef, clusterServiceRef, queryPlanExecutor);
+  }
+
   /**
    * {@inheritDoc} Transform the request and call super.doExecute() to support call from other
    * plugins.
@@ -134,8 +144,9 @@ protected void doExecute(
     QueryContext.setProfile(transformedRequest.profile());
     ActionListener<TransportPPLQueryResponse> clearingListener = wrapWithProfilingClear(listener);
 
-    // Route to analytics engine for non-Lucene (e.g., Parquet-backed) indices
-    if (unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) {
+    // Route to analytics engine for non-Lucene (e.g., Parquet-backed) indices.
+    if (unifiedQueryHandler != null
+        && unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) {
       if (transformedRequest.isExplainRequest()) {
         unifiedQueryHandler.explain(
             transformedRequest.getRequest(),