Roll forward io expansion service to Java17 (#38974)

derrickaw · web-flow · commit e948ef62041f · 2026-06-20T22:57:53.000-04:00
* Second try moving io expansion service to Java17

* fixing xlang issue

* Fix cross-language CI by auto-discovering JDK homes and setting Java 17 as default

* invert java 11 / 17 order installation

* fix gemini

* fix gemini

* decouple BeamModulePlugin and setup-environment-action from GHA infra
diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml
@@ -71,7 +71,7 @@ runs:
 
     - name: Install Java
       if: ${{ inputs.java-version != '' }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }}
diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml
@@ -73,6 +73,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: default
       - name: Set k8s access
         uses: ./.github/actions/setup-k8s-access
@@ -179,6 +182,8 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:io:expansion-service:shadowJar
+          arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64
       # The env variable is created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
       - name: run integrationTest
         uses: ./.github/actions/gradle-command-self-hosted-action
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
@@ -74,6 +74,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: |
             3.10
             3.14
@@ -84,6 +87,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.14 \
             -PuseWheelDistribution \
+            -Pjava17Home=$JAVA_HOME_17_X64 \
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
@@ -74,6 +74,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: |
             3.10
             3.14
@@ -85,6 +88,8 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit
+          arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
@@ -73,6 +73,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: |
             3.10
             3.14
@@ -84,6 +87,7 @@ jobs:
             -PpythonVersion=3.14 \
             -PuseWheelDistribution \
             -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \
+            -Pjava17Home=$JAVA_HOME_17_X64 \
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml
@@ -73,14 +73,19 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: |
             3.10
             3.14
       - name: run PostCommit Python Xlang IO Direct script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit
-          arguments: -PuseWheelDistribution
+          arguments: |
+            -PuseWheelDistribution \
+            -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
@@ -74,7 +74,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
-          java-version: 11
+          java-version: |
+            17
+            11
           python-version: |
             3.10
             ${{ matrix.python_version }}
@@ -83,7 +85,8 @@ jobs:
         with:
           gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava
           arguments: |
-            -PtestJavaVersion=11 \
+            -PtestJavaVersion=17 \
+            -Pjava17Home=$JAVA_HOME_17_X64 \
             -Pjava11Home=$JAVA_HOME_11_X64 \
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml
@@ -74,6 +74,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: ${{ matrix.python_version }}
       - name: run PostCommit XVR Direct script
         env:
@@ -83,6 +86,7 @@ jobs:
           gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
+            -Pjava17Home=$JAVA_HOME_17_X64 \
             -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v7
diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml
@@ -74,13 +74,17 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           python-version: default
-          java-version: '11'
+          java-version: |
+            17
+            11
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db
       - name: run PostCommit Yaml Xlang Direct script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=p310_ml_test,yaml
+          arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml
@@ -102,7 +102,9 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
-          java-version: default
+          java-version: |
+            17
+            11
           python-version: ${{ matrix.python_version }}
       - name: Set PY_VER_CLEAN
         id: set_py_ver_clean
@@ -114,3 +116,5 @@ jobs:
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit --info
+          arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64
diff --git a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml
@@ -87,11 +87,16 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
+          java-version: |
+            17
+            11
           python-version: default
       - name: run PreCommit Yaml Xlang Direct script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:python:yamlIntegrationTests -PbeamPythonExtra=p310_ml_test,yaml
+          arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()
diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml
@@ -109,7 +109,9 @@ jobs:
         if: ${{ startsWith(matrix.container_task, 'java') }}
         uses: ./.github/actions/setup-environment-action
         with:
-          java-version: 11
+          java-version: |
+            17
+            11
       - name: Setup Python environment
         if: ${{ startsWith(matrix.container_task, 'python') }}
         uses: ./.github/actions/setup-environment-action
@@ -120,6 +122,7 @@ jobs:
         with:
           gradle-command: :sdks:${{ matrix.container_task }}
           arguments: |
+            -Pjava17Home=$JAVA_HOME_17_X64 \
             -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \
             -Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION}${LATEST_TAG} \
             -Pcontainer-architecture-list=arm64,amd64 \
diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
@@ -276,11 +276,13 @@ jobs:
           repository: apache/beam
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.1
-      - name: Install Java 11
+      - name: Install Java
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: |
+            17
+            11
       - name: Install Python 3.10
         uses: actions/setup-python@v5
         with:
@@ -570,11 +572,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
-      - name: Install Java 11
+      - name: Install Java 17
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
       - name: Remove default github maven configuration
         # This step is a workaround to avoid a decryption issue of Beam's
         # net.linguica.gradle.maven.settings plugin and github's provided maven
diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml
@@ -111,6 +111,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+      - name: Install Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
       - name: Setup Beam Python
         working-directory: ./sdks/python
         run: |
@@ -174,6 +179,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+      - name: Install Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '17'
       - name: Setup Beam Python
         working-directory: ./sdks/python
         run: |
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1646,6 +1646,12 @@ class BeamModulePlugin implements Plugin<Project> {
           useJUnit()
           executable = "${testJavaHome}/bin/java"
         }
+        // redirect java exec tasks (expansion service, run, shadowJar execs) to specified JDK
+        if (testJavaHome) {
+          project.tasks.withType(JavaExec).configureEach {
+            executable = "${testJavaHome}/bin/java"
+          }
+        }
       }
 
       if (configuration.shadowClosure) {
diff --git a/sdks/java/expansion-service/container/Dockerfile b/sdks/java/expansion-service/container/Dockerfile
@@ -16,7 +16,7 @@
 # limitations under the License.
 ###############################################################################
 
-FROM eclipse-temurin:11
+FROM eclipse-temurin:17
 LABEL Author "Apache Beam <dev@beam.apache.org>"
 ARG TARGETOS
 ARG TARGETARCH
diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle
@@ -25,8 +25,8 @@ applyJavaNature(
   exportJavadoc: false,
   validateShadowJar: false,
   shadowClosure: {},
-  // iceberg requires Java11+
-  requireJavaVersion: JavaVersion.VERSION_11
+  // iceberg requires Java11+ and delta lake requires Java17+
+  requireJavaVersion: JavaVersion.VERSION_17
 )
 
 // We don't want to use the latest version for the entire beam sdk since beam Java users can override it themselves.
