Switch sandbox deps to analytics-api (JDK 21 surface) (#5426)

Author: bowenlan-amzn

.github/workflows/analytics-engine-compat.yml

@@ -0,0 +1,44 @@
+name: Analytics Engine Compatibility
+
+on:
+  pull_request:
+  push:
+    branches-ignore:
+      - 'backport/**'
+      - 'dependabot/**'
+    paths:
+      - '**/*.java'
+      - '**gradle*'
+      - 'integ-test/**'
+      - '.github/workflows/analytics-engine-compat.yml'
+  merge_group:
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  analytics-engine-compat:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+      - uses: actions/checkout@v4
+
+      - name: Set up JDK 25
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 25
+
+      - name: Run analytics-engine compatibility smoke test
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew :integ-test:analyticsEngineCompatIT"

build.gradle

@@ -127,11 +127,6 @@ allprojects {
         version += "-SNAPSHOT"
     }
 
-    // Path to the analytics-engine plugin ZIP. Override with
-    // `-PanalyticsEngineZip=/path/to/zip` if needed.
-    ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?:
-            "${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip"
-
     plugins.withId('java') {
         java {
             sourceCompatibility = JavaVersion.VERSION_21

core/build.gradle

@@ -64,11 +64,11 @@ dependencies {
     }
     api 'org.apache.calcite:calcite-linq4j:1.41.0'
     api project(':common')
-    compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
-    // Needed because the analytics-framework's QueryPlanExecutor signature uses
+    compileOnly 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT'
+    // Needed because analytics-api's QueryPlanExecutor signature uses
     // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type.
     compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
-    testImplementation files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    testImplementation 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT'
     testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
     implementation "com.github.seancfoley:ipaddress:5.4.2"
     implementation "com.jayway.jsonpath:json-path:2.9.0"

doctest/build.gradle

@@ -15,6 +15,7 @@ plugins {
 
 apply plugin: 'opensearch.testclusters'
 
+
 def path = project(':').projectDir
 // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from
 // plugin module, and will only build ppl in it.
@@ -195,7 +196,6 @@ testClusters {
         }))
         */
         plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload))
-        plugin provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) }
         plugin ':opensearch-sql-plugin'
         testDistribution = 'archive'
     }

integ-test/build.gradle

@@ -270,43 +270,72 @@ def getGeoSpatialPlugin() {
     }
 }
 
+// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead).
+ext.pluginVersion = opensearch_version.tokenize('-')[0]
+ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins"
+ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip"
+ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip"
+
+task downloadAnalyticsEngineZip(type: Download) {
+    src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip"
+    dest analyticsEngineZipDest
+    overwrite false
+    onlyIfModified true
+    onlyIf { !project.findProperty('analyticsEngineZip') }
+}
+
+task downloadArrowFlightRpcZip(type: Download) {
+    src "${featureBuildBase}/0-arrow-flight-rpc-${pluginVersion}.zip"
+    dest arrowFlightRpcZipDest
+    overwrite false
+    onlyIfModified true
+    onlyIf { !project.findProperty('arrowFlightRpcZip') }
+}
+
 def getAnalyticsEnginePlugin() {
-    provider { (RegularFile) (() -> file("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip")) }
+    provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) }
+}
+
+def getArrowFlightRpcPlugin() {
+    provider { (RegularFile) (() -> file(project.findProperty('arrowFlightRpcZip') ?: arrowFlightRpcZipDest)) }
 }
 
 testClusters {
     integTest {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
-        plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
     }
     yamlRestTest {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
-        plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
     }
     remoteCluster {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
         plugin(getGeoSpatialPlugin())
-        plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
     integTestWithSecurity {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
-        plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
     remoteIntegTestWithSecurity {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
+        plugin ":opensearch-sql-plugin"
+    }
+    // Smoke test: verify sql loads cleanly alongside analytics-engine.
+    analyticsEngineCompat {
+        testDistribution = 'archive'
+        plugin(getJobSchedulerPlugin())
+        plugin(getArrowFlightRpcPlugin())
         plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
@@ -360,9 +389,17 @@ task stopPrometheus(type: KillProcessTask) {
 
 stopPrometheus.mustRunAfter startPrometheus
 
+task analyticsEngineCompatIT(type: RestIntegTestTask) {
+    useCluster testClusters.analyticsEngineCompat
+    dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip
+    systemProperty 'tests.security.manager', 'false'
+    filter {
+        includeTestsMatching 'org.opensearch.sql.plugin.AnalyticsEngineCompatIT'
+    }
+}
+
 task integJdbcTest(type: RestIntegTestTask) {
     testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with {
-        plugin(getAnalyticsEnginePlugin())
         plugin ":opensearch-sql-plugin"
     }
 

integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.plugin;
+
+import org.junit.Test;
+import org.opensearch.test.rest.OpenSearchRestTestCase;
+
+/**
+ * Smoke test: verifies that opensearch-sql loads cleanly alongside arrow-flight-rpc and
+ * analytics-engine. A successful cluster start is the only assertion — no sql-specific logic runs.
+ */
+public class AnalyticsEngineCompatIT extends OpenSearchRestTestCase {
+
+  @Test
+  public void testClusterStarted() {
+    // If the cluster booted, all three plugins loaded without classloader errors.
+  }
+}

libs/analytics-engine-3.7.0-SNAPSHOT.jar

@@ -1,5 +1,4 @@
 import java.util.concurrent.Callable
-import org.opensearch.gradle.dependencies.CompileOnlyResolvePlugin
 
 /*
  * Copyright OpenSearch Contributors
@@ -161,10 +160,7 @@ dependencies {
 
     api project(":ppl")
     api project(':api')
-    // Bundled: analytics-framework interfaces must resolve even when the plugin is absent.
-    api files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
-    // Not bundled: classes here (e.g. OpenSearchSchemaBuilder) only load when the plugin is installed.
-    compileOnly files("${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.jar")
+    implementation("org.opensearch.sandbox:analytics-api:${opensearch_version}")
     api project(':legacy')
     api project(':opensearch')
     api project(':prometheus')
@@ -310,7 +306,6 @@ def getJobSchedulerPlugin() {
 
 testClusters.integTest {
     plugin(getJobSchedulerPlugin())
-    plugin provider { (RegularFile) (() -> file("${analyticsEngineZip}")) }
     plugin(project.tasks.bundlePlugin.archiveFile)
     testDistribution = "ARCHIVE"
 
@@ -326,4 +321,3 @@ testClusters.integTest {
 run {
     useCluster testClusters.integTest
 }
-