Skip to content

Commit 2aa3351

Browse files
AbacnAbacn
authored
Fix Java8 tests to have gradle run on Java11 (#35609)
* Fix Java8 tests to have gradle run on Java11 * remove extra build jar * Fix Direct JavaVersion typo, clean up * Fix typo
1 parent 4dddc19 commit 2aa3351

7 files changed

Lines changed: 20 additions & 76 deletions

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -80,24 +80,13 @@ jobs:
8080
java-version: |
8181
${{ matrix.java_version }}
8282
11
83-
- name: run jar Java${{ matrix.java_version }} script
84-
run: |
85-
./gradlew runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar \
86-
-Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64
8783
- name: run validatesRunner Java${{ matrix.java_version }} script
8884
uses: ./.github/actions/gradle-command-self-hosted-action
8985
with:
9086
gradle-command: :runners:google-cloud-dataflow-java:validatesRunner
9187
arguments: |
92-
-x shadowJar \
93-
-x shadowTestJar \
94-
-x compileJava \
95-
-x compileTestJava \
96-
-x jar \
97-
-x testJar \
98-
-x classes \
99-
-x testClasses \
100-
-Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
88+
-PtestJavaVersion=${{ matrix.java_version }} \
89+
-Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
10190
max-workers: 12
10291
- name: Archive JUnit Test Results
10392
uses: actions/upload-artifact@v4

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,20 +80,13 @@ jobs:
8080
java-version: |
8181
${{ matrix.java_version }}
8282
11
83-
- name: run jar Java${{ matrix.java_version }} script
84-
run: |
85-
./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \
86-
-Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64
8783
- name: run validatesRunner Java${{ matrix.java_version }} script
8884
uses: ./.github/actions/gradle-command-self-hosted-action
8985
with:
9086
gradle-command: :runners:direct-java:validatesRunner
9187
arguments: |
92-
-x shadowJar \
93-
-x shadowTestJar \
94-
-x compileJava \
95-
-x compileTestJava \
96-
-Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
88+
-PtestJavaVersion=${{ matrix.java_version }} \
89+
-Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
9790
- name: Archive JUnit Test Results
9891
uses: actions/upload-artifact@v4
9992
if: ${{ !success() }}

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,23 +78,13 @@ jobs:
7878
java-version: |
7979
8
8080
11
81-
- name: run jar Java8 script
82-
run: |
83-
./gradlew :runners:flink:1.19:jar :runners:flink:1.19:testJar
8481
- name: run validatesRunner Java8 script
8582
uses: ./.github/actions/gradle-command-self-hosted-action
8683
with:
8784
gradle-command: :runners:flink:1.19:validatesRunner
8885
arguments: |
89-
-x shadowJar \
90-
-x shadowTestJar \
91-
-x compileJava \
92-
-x compileTestJava \
93-
-x jar \
94-
-x testJar \
95-
-x classes \
96-
-x testClasses \
97-
-Dorg.gradle.java.home=$JAVA_HOME_8_X64 \
86+
-PtestJavaVersion=8 \
87+
-Pjava8Home=$JAVA_HOME_8_X64 \
9888
max-workers: 12
9989
- name: Archive JUnit Test Results
10090
uses: actions/upload-artifact@v4

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,23 +78,13 @@ jobs:
7878
java-version: |
7979
8
8080
11
81-
- name: run jar Java8 script
82-
run: |
83-
./gradlew :runners:spark:3:jar :runners:spark:3:testJar
8481
- name: run validatesRunner Java8 script
8582
uses: ./.github/actions/gradle-command-self-hosted-action
8683
with:
8784
gradle-command: :runners:spark:3:validatesRunner
8885
arguments: |
89-
-x shadowJar \
90-
-x shadowTestJar \
91-
-x compileJava \
92-
-x compileTestJava \
93-
-x jar \
94-
-x testJar \
95-
-x classes \
96-
-x testClasses \
97-
-Dorg.gradle.java.home=$JAVA_HOME_8_X64 \
86+
-PtestJavaVersion=8 \
87+
-Pjava8Home=$JAVA_HOME_8_X64 \
9888
max-workers: 12
9989
- name: Archive JUnit Test Results
10090
uses: actions/upload-artifact@v4

‎buildSrc/build.gradle.kts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,9 +92,5 @@ gradlePlugin {
9292
id = "org.apache.beam.vendor-java"
9393
implementationClass = "org.apache.beam.gradle.VendorJavaPlugin"
9494
}
95-
create("beamJenkins") {
96-
id = "org.apache.beam.jenkins"
97-
implementationClass = "org.apache.beam.gradle.BeamJenkinsPlugin"
98-
}
9995
}
10096
}

‎sdks/go/test/build.gradle‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ task dataflowValidatesRunner() {
4141
]
4242
exec {
4343
if (fork_java_home) {
44-
environment "JAVA_HOME", fork_java_home
44+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
4545
}
4646
executable "sh"
4747
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -66,7 +66,7 @@ task dataflowValidatesRunnerARM64() {
6666
]
6767
exec {
6868
if (fork_java_home) {
69-
environment "JAVA_HOME", fork_java_home
69+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
7070
}
7171
executable "sh"
7272
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -95,7 +95,7 @@ task flinkValidatesRunner {
9595
]
9696
exec {
9797
if (fork_java_home) {
98-
environment "JAVA_HOME", fork_java_home
98+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
9999
}
100100
executable "sh"
101101
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -123,7 +123,7 @@ task samzaValidatesRunner {
123123
]
124124
exec {
125125
if (fork_java_home) {
126-
environment "JAVA_HOME", fork_java_home
126+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
127127
}
128128
executable "sh"
129129
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -151,7 +151,7 @@ task sparkValidatesRunner {
151151
]
152152
exec {
153153
if (fork_java_home) {
154-
environment "JAVA_HOME", fork_java_home
154+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
155155
}
156156
executable "sh"
157157
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -190,7 +190,7 @@ tasks.register("ulrValidatesRunner") {
190190
}
191191
exec {
192192
if (fork_java_home) {
193-
environment "JAVA_HOME", fork_java_home
193+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
194194
}
195195
executable "sh"
196196
args "-c", ". ${envdir}/bin/activate && ./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -217,7 +217,7 @@ task prismValidatesRunner {
217217
]
218218
exec {
219219
if (fork_java_home) {
220-
environment "JAVA_HOME", fork_java_home
220+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
221221
}
222222
executable "sh"
223223
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"
@@ -262,7 +262,7 @@ ext.goIoValidatesRunnerTask = { proj, name, scriptOpts, pipelineOpts ->
262262
logger.info("Running the command: sh -c ./run_validatesrunner_tests.sh ${options.join(' ')}")
263263
exec {
264264
if (fork_java_home) {
265-
environment "JAVA_HOME", fork_java_home
265+
environment "JAVA_HOME_JOB_SERVER", fork_java_home
266266
}
267267
executable "sh"
268268
args "-c", "./run_validatesrunner_tests.sh ${options.join(' ')}"

‎sdks/go/test/run_validatesrunner_tests.sh‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -259,8 +259,8 @@ s.close()
259259

260260
TMPDIR=$(mktemp -d)
261261

262-
if [[ -n "$JAVA_HOME" ]]; then
263-
JAVA_CMD="$JAVA_HOME/bin/java"
262+
if [[ -n "$JAVA_HOME_JOB_SERVER" ]]; then
263+
JAVA_CMD="$JAVA_HOME_JOB_SERVER/bin/java"
264264
else
265265
JAVA_CMD="java"
266266
fi
@@ -374,18 +374,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then
374374
CONTAINER=us.gcr.io/$PROJECT/$USER/beam_go_sdk
375375
echo "Using container $CONTAINER"
376376

377-
# TODO(https://github.com/apache/beam/issues/27674): remove this branch once the jenkins VM can build multiarch, or jenkins is deprecated.
378-
if [[ "$USER" == "jenkins" ]]; then
379-
./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG
380-
381-
# Verify it exists
382-
docker images | grep $TAG
383-
384-
# Push the container
385-
gcloud docker -- push $CONTAINER:$TAG
386-
else
387-
./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers
388-
fi
377+
./gradlew :sdks:go:container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Pcontainer-architecture-list=arm64,amd64 -Ppush-containers
389378

390379
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then
391380
ARGS="$ARGS --experiments=use_portable_job_submission"
@@ -395,7 +384,7 @@ if [[ "$RUNNER" == "dataflow" ]]; then
395384
JAVA_TAG=$(date +%Y%m%d-%H%M%S)
396385
JAVA_CONTAINER=us.gcr.io/$PROJECT/$USER/beam_java11_sdk
397386
echo "Using container $JAVA_CONTAINER for cross-language java transforms"
398-
./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG -Pjava11Home=$JAVA11_HOME
387+
./gradlew :sdks:java:container:java11:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$JAVA_TAG
399388

400389
# Verify it exists
401390
docker images | grep $JAVA_TAG
@@ -457,9 +446,6 @@ if [[ "$RUNNER" == "dataflow" ]]; then
457446
# Note: we don't delete the multi-arch containers here because this command only deletes the manifest list with the tag,
458447
# the associated container images can't be deleted because they are not tagged. However, multi-arch containers that are
459448
# older than 6 weeks old are deleted by stale_dataflow_prebuilt_image_cleaner.sh that runs daily.
460-
if [[ "$USER" == "jenkins" ]]; then
461-
gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
462-
fi
463449
if [[ -n "$TEST_EXPANSION_ADDR" || -n "$IO_EXPANSION_ADDR" || -n "$SCHEMAIO_EXPANSION_ADDR" || -n "$DEBEZIUMIO_EXPANSION_ADDR" ]]; then
464450
# Delete the java cross-language container locally and remotely
465451
docker rmi $JAVA_CONTAINER:$JAVA_TAG || echo "Failed to remove container"

0 commit comments

Comments
 (0)