From 2497e55cd7e374f8cb19d87655bdc13c46069952 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 11 May 2026 16:09:09 -0400 Subject: [PATCH] Exercise Spark 4 tests --- .github/workflows/README.md | 2 +- ...=> beam_PreCommit_Java_Spark_Versions.yml} | 21 ++- .../beam/gradle/BeamModulePlugin.groovy | 139 +++++++++--------- 3 files changed, 82 insertions(+), 80 deletions(-) rename .github/workflows/{beam_PreCommit_Java_Spark3_Versions.yml => beam_PreCommit_Java_Spark_Versions.yml} (86%) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c3c73c0317a9..9867810a7419 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -267,7 +267,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit Java SingleStore IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml) | N/A |`Run Java_SingleStore_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Snowflake IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml) | N/A |`Run Java_Snowflake_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Solr IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml) | N/A |`Run Java_Solr_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml?query=event%3Aschedule) | -| [ PreCommit Java Spark3 Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml) | N/A | `Run Java_Spark3_Versions PreCommit` | [![.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml?query=event%3Aschedule) | +| [ PreCommit Java Spark Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml) | N/A | `Run Java_Spark_Versions PreCommit` | [![.github/workflows/beam_PreCommit_Java_Spark_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml?query=event%3Aschedule) | | [ PreCommit Java Splunk IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml) | N/A |`Run Java_Splunk_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Thrift IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml) | N/A |`Run Java_Thrift_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml?query=event%3Aschedule) | | [ PreCommit Java Tika IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml) | N/A |`Run Java_Tika_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml?query=event%3Aschedule) | diff --git a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml similarity index 86% rename from .github/workflows/beam_PreCommit_Java_Spark3_Versions.yml rename to .github/workflows/beam_PreCommit_Java_Spark_Versions.yml index dd0bb69b32fe..6f8689d00caa 100644 --- a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml +++ b/.github/workflows/beam_PreCommit_Java_Spark_Versions.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: PreCommit Java Spark3 Versions +name: PreCommit Java Spark Versions on: push: @@ -23,13 +23,13 @@ on: branches: ['master', 'release-*'] paths: - 'runners/spark/**' - - '.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml' + - '.github/workflows/beam_PreCommit_Java_Spark_Versions.yml' pull_request_target: branches: ['master', 'release-*'] paths: - 'runners/spark/**' - 'release/trigger_all_tests.json' - - '.github/trigger_files/beam_PreCommit_Java_Spark3_Versions.json' + - '.github/trigger_files/beam_PreCommit_Java_Spark_Versions.json' issue_comment: types: [created] schedule: @@ -63,20 +63,20 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PreCommit_Java_Spark3_Versions: + beam_PreCommit_Java_Spark_Versions: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) runs-on: [self-hosted, ubuntu-24.04, main] strategy: matrix: - job_name: [beam_PreCommit_Java_Spark3_Versions] - job_phrase: [Run Java_Spark3_Versions PreCommit] + job_name: [beam_PreCommit_Java_Spark_Versions] + job_phrase: [Run Java_Spark_Versions PreCommit] timeout-minutes: 120 if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || github.event_name == 'workflow_dispatch' || - github.event.comment.body == 'Run Java_Spark3_Versions PreCommit' + github.event.comment.body == 'Run Java_Spark_Versions PreCommit' steps: - uses: actions/checkout@v6 - name: Setup repository @@ -87,12 +87,17 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: | + 17 + 11 - name: run sparkVersionsTest script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :runners:spark:3:sparkVersionsTest + gradle-command: :runners:spark:3:sparkVersionsTest :runners:spark:4:build arguments: | -PdisableSpotlessCheck=true \ + -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 if: ${{ !success() }} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index e32d1c9afb77..2e064b367db3 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1516,72 +1516,66 @@ class BeamModulePlugin implements Plugin { project.tasks.analyzeDependencies.enabled = false } - // errorprone requires java9+ compiler. It can be used with Java8 but then sets a java9+ errorproneJavac. - // However, the redirect ignores any task that forks and defines either a javaHome or an executable, - // see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support - // which means errorprone cannot run when gradle runs on Java11+ but serve `-testJavaVersion=8 -Pjava8Home` options - if (!(project.findProperty('testJavaVersion') == '8')) { - // Enable errorprone static analysis - project.apply plugin: 'net.ltgt.errorprone' + // Enable errorprone static analysis + project.apply plugin: 'net.ltgt.errorprone' - project.dependencies { - errorprone("com.google.errorprone:error_prone_core:$errorprone_version") - errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28") - } - - project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } - - project.tasks.withType(JavaCompile) { - options.errorprone.disableWarningsInGeneratedCode = true - options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' - - // Error Prone requires some packages to be exported/opened on Java versions that support modules, - // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. - // The -J prefix is not needed if forkOptions.javaHome is unset, - // see http://github.com/gradle/gradle/issues/22747 - if (options.forkOptions.javaHome == null) { - options.fork = true - options.forkOptions.jvmArgs += errorProneAddModuleOpts - } - def disabledChecks = [ - // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks - "AutoValueImmutableFields", - "ComparableType", - "DoNotMockAutoValue", - "EmptyBlockTag", - "ExtendsAutoValue", - "InlineMeSuggester", - "InvalidBlockTag", - "JodaConstructors", - "MixedMutabilityReturnType", - "PreferJavaTimeOverload", - "Slf4jSignOnlyFormat", - "UnrecognisedJavadocTag", - // errorprone 3.2.0+ checks - "DirectInvocationOnMock", - "MockNotUsedInProduction", - "NullableWildcard", - "SuperCallToObjectMethod", - // Intended suppressions with justifications - // for encoding efficiency and backward compatibility - "EnumOrdinal", - // widely used in non-public methods - "NotJavadoc", - // return values used for assignments widely, and for backward compatibility. - "NonApiType", - // Used to test self equal - "SelfAssertion", - // Sometimes a static logger is preferred, which is the convention currently used in beam. See docs: - // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static - "Slf4jLoggerShouldBeNonStatic", - // allow implicit Locale.Default - "StringCaseLocaleUsage", - // DoFn methods are executed reflectively at pipeline runtime - "UnusedMethod", - ] - disabledChecks.each { - options.errorprone.errorproneArgs.add("-Xep:${it}:OFF") - } + project.dependencies { + errorprone("com.google.errorprone:error_prone_core:$errorprone_version") + errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28") + } + + project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } + + project.tasks.withType(JavaCompile) { + options.errorprone.disableWarningsInGeneratedCode = true + options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' + + // Error Prone requires some packages to be exported/opened on Java versions that support modules, + // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. + // The -J prefix is not needed if forkOptions.javaHome is unset, + // see http://github.com/gradle/gradle/issues/22747 + if (options.forkOptions.javaHome == null) { + options.fork = true + options.forkOptions.jvmArgs += errorProneAddModuleOpts + } + def disabledChecks = [ + // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks + "AutoValueImmutableFields", + "ComparableType", + "DoNotMockAutoValue", + "EmptyBlockTag", + "ExtendsAutoValue", + "InlineMeSuggester", + "InvalidBlockTag", + "JodaConstructors", + "MixedMutabilityReturnType", + "PreferJavaTimeOverload", + "Slf4jSignOnlyFormat", + "UnrecognisedJavadocTag", + // errorprone 3.2.0+ checks + "DirectInvocationOnMock", + "MockNotUsedInProduction", + "NullableWildcard", + "SuperCallToObjectMethod", + // Intended suppressions with justifications + // for encoding efficiency and backward compatibility + "EnumOrdinal", + // widely used in non-public methods + "NotJavadoc", + // return values used for assignments widely, and for backward compatibility. + "NonApiType", + // Used to test self equal + "SelfAssertion", + // Sometimes a static logger is preferred, which is the convention currently used in beam. See docs: + // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static + "Slf4jLoggerShouldBeNonStatic", + // allow implicit Locale.Default + "StringCaseLocaleUsage", + // DoFn methods are executed reflectively at pipeline runtime + "UnusedMethod", + ] + disabledChecks.each { + options.errorprone.errorproneArgs.add("-Xep:${it}:OFF") } } @@ -1624,16 +1618,19 @@ class BeamModulePlugin implements Plugin { preserveFileTimestamps(false) } + String testJavaVersion = project.findProperty('testJavaVersion') + if (!testJavaVersion && forkJavaVersion) { + testJavaVersion = forkJavaVersion + } // if specified test java version, modify the compile and runtime versions accordingly - if (['11', '17', '21', '25'].contains(project.findProperty('testJavaVersion'))) { - String ver = project.getProperty('testJavaVersion') - def testJavaHome = project.getProperty("java${ver}Home") + if (['11', '17', '21', '25'].contains(testJavaVersion)) { + def testJavaHome = project.getProperty("java${testJavaVersion}Home") // redirect java compiler to specified version for compileTestJava only project.tasks.compileTestJava { - setCompileAndRuntimeJavaVersion(options.compilerArgs, ver) - project.ext.setJavaVerOptions(options, ver) - if (ver == '25') { + setCompileAndRuntimeJavaVersion(options.compilerArgs, testJavaVersion) + project.ext.setJavaVerOptions(options, testJavaVersion) + if (testJavaVersion == '25') { // TODO: Upgrade errorprone version to support Java25. Currently compile crashes // java.lang.NoSuchFieldError: Class com.sun.tools.javac.code.TypeTag does not have member field // 'com.sun.tools.javac.code.TypeTag UNKNOWN'