Skip to content

Commit 4d683c0

Browse files
authored
Exercise Spark 4 tests (#38453)
1 parent 34c6e26 commit 4d683c0

3 files changed

Lines changed: 82 additions & 80 deletions

File tree

.github/workflows/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
267267
| [ PreCommit Java SingleStore IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml) | N/A |`Run Java_SingleStore_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml?query=event%3Aschedule) |
268268
| [ PreCommit Java Snowflake IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml) | N/A |`Run Java_Snowflake_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml?query=event%3Aschedule) |
269269
| [ PreCommit Java Solr IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml) | N/A |`Run Java_Solr_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml?query=event%3Aschedule) |
270-
| [ PreCommit Java Spark3 Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml) | N/A | `Run Java_Spark3_Versions PreCommit` | [![.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark3_Versions.yml?query=event%3Aschedule) |
270+
| [ PreCommit Java Spark Versions ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml) | N/A | `Run Java_Spark_Versions PreCommit` | [![.github/workflows/beam_PreCommit_Java_Spark_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Spark_Versions.yml?query=event%3Aschedule) |
271271
| [ PreCommit Java Splunk IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml) | N/A |`Run Java_Splunk_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml?query=event%3Aschedule) |
272272
| [ PreCommit Java Thrift IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml) | N/A |`Run Java_Thrift_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml?query=event%3Aschedule) |
273273
| [ PreCommit Java Tika IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml) | N/A |`Run Java_Tika_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml?query=event%3Aschedule) |

.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml renamed to .github/workflows/beam_PreCommit_Java_Spark_Versions.yml

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,21 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
name: PreCommit Java Spark3 Versions
18+
name: PreCommit Java Spark Versions
1919

2020
on:
2121
push:
2222
tags: ['v*']
2323
branches: ['master', 'release-*']
2424
paths:
2525
- 'runners/spark/**'
26-
- '.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml'
26+
- '.github/workflows/beam_PreCommit_Java_Spark_Versions.yml'
2727
pull_request_target:
2828
branches: ['master', 'release-*']
2929
paths:
3030
- 'runners/spark/**'
3131
- 'release/trigger_all_tests.json'
32-
- '.github/trigger_files/beam_PreCommit_Java_Spark3_Versions.json'
32+
- '.github/trigger_files/beam_PreCommit_Java_Spark_Versions.json'
3333
issue_comment:
3434
types: [created]
3535
schedule:
@@ -63,20 +63,20 @@ env:
6363
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
6464

6565
jobs:
66-
beam_PreCommit_Java_Spark3_Versions:
66+
beam_PreCommit_Java_Spark_Versions:
6767
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
6868
runs-on: [self-hosted, ubuntu-24.04, main]
6969
strategy:
7070
matrix:
71-
job_name: [beam_PreCommit_Java_Spark3_Versions]
72-
job_phrase: [Run Java_Spark3_Versions PreCommit]
71+
job_name: [beam_PreCommit_Java_Spark_Versions]
72+
job_phrase: [Run Java_Spark_Versions PreCommit]
7373
timeout-minutes: 120
7474
if: |
7575
github.event_name == 'push' ||
7676
github.event_name == 'pull_request_target' ||
7777
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
7878
github.event_name == 'workflow_dispatch' ||
79-
github.event.comment.body == 'Run Java_Spark3_Versions PreCommit'
79+
github.event.comment.body == 'Run Java_Spark_Versions PreCommit'
8080
steps:
8181
- uses: actions/checkout@v6
8282
- name: Setup repository
@@ -87,12 +87,17 @@ jobs:
8787
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
8888
- name: Setup environment
8989
uses: ./.github/actions/setup-environment-action
90+
with:
91+
java-version: |
92+
17
93+
11
9094
- name: run sparkVersionsTest script
9195
uses: ./.github/actions/gradle-command-self-hosted-action
9296
with:
93-
gradle-command: :runners:spark:3:sparkVersionsTest
97+
gradle-command: :runners:spark:3:sparkVersionsTest :runners:spark:4:build
9498
arguments: |
9599
-PdisableSpotlessCheck=true \
100+
-Pjava17Home=$JAVA_HOME_17_X64
96101
- name: Archive JUnit Test Results
97102
uses: actions/upload-artifact@v7
98103
if: ${{ !success() }}

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

Lines changed: 68 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,72 +1516,66 @@ class BeamModulePlugin implements Plugin<Project> {
15161516
project.tasks.analyzeDependencies.enabled = false
15171517
}
15181518

1519-
// errorprone requires java9+ compiler. It can be used with Java8 but then sets a java9+ errorproneJavac.
1520-
// However, the redirect ignores any task that forks and defines either a javaHome or an executable,
1521-
// see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support
1522-
// which means errorprone cannot run when gradle runs on Java11+ but serve `-testJavaVersion=8 -Pjava8Home` options
1523-
if (!(project.findProperty('testJavaVersion') == '8')) {
1524-
// Enable errorprone static analysis
1525-
project.apply plugin: 'net.ltgt.errorprone'
1519+
// Enable errorprone static analysis
1520+
project.apply plugin: 'net.ltgt.errorprone'
15261521

1527-
project.dependencies {
1528-
errorprone("com.google.errorprone:error_prone_core:$errorprone_version")
1529-
errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28")
1530-
}
1531-
1532-
project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" }
1533-
1534-
project.tasks.withType(JavaCompile) {
1535-
options.errorprone.disableWarningsInGeneratedCode = true
1536-
options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*'
1537-
1538-
// Error Prone requires some packages to be exported/opened on Java versions that support modules,
1539-
// i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403.
1540-
// The -J prefix is not needed if forkOptions.javaHome is unset,
1541-
// see http://github.com/gradle/gradle/issues/22747
1542-
if (options.forkOptions.javaHome == null) {
1543-
options.fork = true
1544-
options.forkOptions.jvmArgs += errorProneAddModuleOpts
1545-
}
1546-
def disabledChecks = [
1547-
// TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks
1548-
"AutoValueImmutableFields",
1549-
"ComparableType",
1550-
"DoNotMockAutoValue",
1551-
"EmptyBlockTag",
1552-
"ExtendsAutoValue",
1553-
"InlineMeSuggester",
1554-
"InvalidBlockTag",
1555-
"JodaConstructors",
1556-
"MixedMutabilityReturnType",
1557-
"PreferJavaTimeOverload",
1558-
"Slf4jSignOnlyFormat",
1559-
"UnrecognisedJavadocTag",
1560-
// errorprone 3.2.0+ checks
1561-
"DirectInvocationOnMock",
1562-
"MockNotUsedInProduction",
1563-
"NullableWildcard",
1564-
"SuperCallToObjectMethod",
1565-
// Intended suppressions with justifications
1566-
// for encoding efficiency and backward compatibility
1567-
"EnumOrdinal",
1568-
// widely used in non-public methods
1569-
"NotJavadoc",
1570-
// return values used for assignments widely, and for backward compatibility.
1571-
"NonApiType",
1572-
// Used to test self equal
1573-
"SelfAssertion",
1574-
// Sometimes a static logger is preferred, which is the convention currently used in beam. See docs:
1575-
// https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static
1576-
"Slf4jLoggerShouldBeNonStatic",
1577-
// allow implicit Locale.Default
1578-
"StringCaseLocaleUsage",
1579-
// DoFn methods are executed reflectively at pipeline runtime
1580-
"UnusedMethod",
1581-
]
1582-
disabledChecks.each {
1583-
options.errorprone.errorproneArgs.add("-Xep:${it}:OFF")
1584-
}
1522+
project.dependencies {
1523+
errorprone("com.google.errorprone:error_prone_core:$errorprone_version")
1524+
errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.28")
1525+
}
1526+
1527+
project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" }
1528+
1529+
project.tasks.withType(JavaCompile) {
1530+
options.errorprone.disableWarningsInGeneratedCode = true
1531+
options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*'
1532+
1533+
// Error Prone requires some packages to be exported/opened on Java versions that support modules,
1534+
// i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403.
1535+
// The -J prefix is not needed if forkOptions.javaHome is unset,
1536+
// see http://github.com/gradle/gradle/issues/22747
1537+
if (options.forkOptions.javaHome == null) {
1538+
options.fork = true
1539+
options.forkOptions.jvmArgs += errorProneAddModuleOpts
1540+
}
1541+
def disabledChecks = [
1542+
// TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks
1543+
"AutoValueImmutableFields",
1544+
"ComparableType",
1545+
"DoNotMockAutoValue",
1546+
"EmptyBlockTag",
1547+
"ExtendsAutoValue",
1548+
"InlineMeSuggester",
1549+
"InvalidBlockTag",
1550+
"JodaConstructors",
1551+
"MixedMutabilityReturnType",
1552+
"PreferJavaTimeOverload",
1553+
"Slf4jSignOnlyFormat",
1554+
"UnrecognisedJavadocTag",
1555+
// errorprone 3.2.0+ checks
1556+
"DirectInvocationOnMock",
1557+
"MockNotUsedInProduction",
1558+
"NullableWildcard",
1559+
"SuperCallToObjectMethod",
1560+
// Intended suppressions with justifications
1561+
// for encoding efficiency and backward compatibility
1562+
"EnumOrdinal",
1563+
// widely used in non-public methods
1564+
"NotJavadoc",
1565+
// return values used for assignments widely, and for backward compatibility.
1566+
"NonApiType",
1567+
// Used to test self equal
1568+
"SelfAssertion",
1569+
// Sometimes a static logger is preferred, which is the convention currently used in beam. See docs:
1570+
// https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static
1571+
"Slf4jLoggerShouldBeNonStatic",
1572+
// allow implicit Locale.Default
1573+
"StringCaseLocaleUsage",
1574+
// DoFn methods are executed reflectively at pipeline runtime
1575+
"UnusedMethod",
1576+
]
1577+
disabledChecks.each {
1578+
options.errorprone.errorproneArgs.add("-Xep:${it}:OFF")
15851579
}
15861580
}
15871581

@@ -1624,16 +1618,19 @@ class BeamModulePlugin implements Plugin<Project> {
16241618
preserveFileTimestamps(false)
16251619
}
16261620

1621+
String testJavaVersion = project.findProperty('testJavaVersion')
1622+
if (!testJavaVersion && forkJavaVersion) {
1623+
testJavaVersion = forkJavaVersion
1624+
}
16271625
// if specified test java version, modify the compile and runtime versions accordingly
1628-
if (['11', '17', '21', '25'].contains(project.findProperty('testJavaVersion'))) {
1629-
String ver = project.getProperty('testJavaVersion')
1630-
def testJavaHome = project.getProperty("java${ver}Home")
1626+
if (['11', '17', '21', '25'].contains(testJavaVersion)) {
1627+
def testJavaHome = project.getProperty("java${testJavaVersion}Home")
16311628

16321629
// redirect java compiler to specified version for compileTestJava only
16331630
project.tasks.compileTestJava {
1634-
setCompileAndRuntimeJavaVersion(options.compilerArgs, ver)
1635-
project.ext.setJavaVerOptions(options, ver)
1636-
if (ver == '25') {
1631+
setCompileAndRuntimeJavaVersion(options.compilerArgs, testJavaVersion)
1632+
project.ext.setJavaVerOptions(options, testJavaVersion)
1633+
if (testJavaVersion == '25') {
16371634
// TODO: Upgrade errorprone version to support Java25. Currently compile crashes
16381635
// java.lang.NoSuchFieldError: Class com.sun.tools.javac.code.TypeTag does not have member field
16391636
// 'com.sun.tools.javac.code.TypeTag UNKNOWN'

0 commit comments

Comments
 (0)