Skip to content

Commit ba9bec9

Browse files
tkaymakclaude
tkaymak
and
claude
committed
ci: add Spark 4 PreCommit and PostCommit workflows
Add GitHub Actions workflows for the Spark 4 runner module: - beam_PreCommit_Java_Spark4_Versions: runs sparkVersionsTest on changes to runners/spark/**. Currently a no-op (the sparkVersions map is empty) but scaffolds future patch version coverage. - beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming: runs the structured streaming test suite on Java 17. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 83451d7 commit ba9bec9

4 files changed

Lines changed: 215 additions & 0 deletions

‎.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run"
3+
}

‎.github/trigger_files/beam_PreCommit_Java_Spark4_Versions.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run"
3+
}

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PostCommit Java ValidatesRunner Spark4 StructuredStreaming
17+
18+
on:
19+
schedule:
20+
- cron: '45 4/6 * * *'
21+
pull_request_target:
22+
paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming.json']
23+
workflow_dispatch:
24+
25+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: write
29+
checks: write
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: write
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
44+
cancel-in-progress: true
45+
46+
env:
47+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
51+
jobs:
52+
beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming:
53+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
54+
runs-on: [self-hosted, ubuntu-24.04, main]
55+
timeout-minutes: 120
56+
strategy:
57+
matrix:
58+
job_name: [beam_PostCommit_Java_ValidatesRunner_Spark4StructuredStreaming]
59+
job_phrase: [Run Spark4 StructuredStreaming ValidatesRunner]
60+
if: |
61+
github.event_name == 'workflow_dispatch' ||
62+
github.event_name == 'pull_request_target' ||
63+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
64+
github.event.comment.body == 'Run Spark4 StructuredStreaming ValidatesRunner'
65+
steps:
66+
- uses: actions/checkout@v4
67+
- name: Setup repository
68+
uses: ./.github/actions/setup-action
69+
with:
70+
comment_phrase: ${{ matrix.job_phrase }}
71+
github_token: ${{ secrets.GITHUB_TOKEN }}
72+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
73+
- name: Setup environment
74+
uses: ./.github/actions/setup-environment-action
75+
with:
76+
java-version: '17'
77+
- name: run validatesStructuredStreamingRunnerBatch script
78+
uses: ./.github/actions/gradle-command-self-hosted-action
79+
with:
80+
gradle-command: :runners:spark:4:validatesStructuredStreamingRunnerBatch
81+
arguments: |
82+
-PtestJavaVersion=17 \
83+
-PdisableSpotlessCheck=true \
84+
- name: Archive JUnit Test Results
85+
uses: actions/upload-artifact@v4
86+
if: ${{ !success() }}
87+
with:
88+
name: JUnit Test Results
89+
path: "**/build/reports/tests/"
90+
- name: Publish JUnit Test Results
91+
uses: EnricoMi/publish-unit-test-result-action@v2
92+
if: always()
93+
with:
94+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
95+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
96+
files: '**/build/test-results/**/*.xml'
97+
large_files: true

‎.github/workflows/beam_PreCommit_Java_Spark4_Versions.yml‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PreCommit Java Spark4 Versions
19+
20+
on:
21+
push:
22+
tags: ['v*']
23+
branches: ['master', 'release-*']
24+
paths:
25+
- 'runners/spark/**'
26+
- '.github/workflows/beam_PreCommit_Java_Spark4_Versions.yml'
27+
pull_request_target:
28+
branches: ['master', 'release-*']
29+
paths:
30+
- 'runners/spark/**'
31+
- 'release/trigger_all_tests.json'
32+
- '.github/trigger_files/beam_PreCommit_Java_Spark4_Versions.json'
33+
issue_comment:
34+
types: [created]
35+
schedule:
36+
- cron: '30 2/6 * * *'
37+
workflow_dispatch:
38+
39+
# This allows a subsequently queued workflow run to interrupt previous runs
40+
concurrency:
41+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
42+
cancel-in-progress: true
43+
44+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
45+
permissions:
46+
actions: write
47+
pull-requests: write
48+
checks: write
49+
contents: read
50+
deployments: read
51+
id-token: none
52+
issues: write
53+
discussions: read
54+
packages: read
55+
pages: read
56+
repository-projects: read
57+
security-events: read
58+
statuses: read
59+
60+
env:
61+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
62+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
63+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
64+
65+
jobs:
66+
beam_PreCommit_Java_Spark4_Versions:
67+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
68+
runs-on: [self-hosted, ubuntu-24.04, main]
69+
strategy:
70+
matrix:
71+
job_name: [beam_PreCommit_Java_Spark4_Versions]
72+
job_phrase: [Run Java_Spark4_Versions PreCommit]
73+
timeout-minutes: 120
74+
if: |
75+
github.event_name == 'push' ||
76+
github.event_name == 'pull_request_target' ||
77+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
78+
github.event_name == 'workflow_dispatch' ||
79+
github.event.comment.body == 'Run Java_Spark4_Versions PreCommit'
80+
steps:
81+
- uses: actions/checkout@v4
82+
- name: Setup repository
83+
uses: ./.github/actions/setup-action
84+
with:
85+
comment_phrase: ${{ matrix.job_phrase }}
86+
github_token: ${{ secrets.GITHUB_TOKEN }}
87+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
88+
- name: Setup environment
89+
uses: ./.github/actions/setup-environment-action
90+
with:
91+
java-version: '17'
92+
- name: run sparkVersionsTest script
93+
uses: ./.github/actions/gradle-command-self-hosted-action
94+
with:
95+
gradle-command: :runners:spark:4:sparkVersionsTest
96+
arguments: |
97+
-PtestJavaVersion=17 \
98+
-PdisableSpotlessCheck=true \
99+
- name: Archive JUnit Test Results
100+
uses: actions/upload-artifact@v4
101+
if: ${{ !success() }}
102+
with:
103+
name: JUnit Test Results
104+
path: "**/build/reports/tests/"
105+
- name: Publish JUnit Test Results
106+
uses: EnricoMi/publish-unit-test-result-action@v2
107+
if: always()
108+
with:
109+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
110+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
111+
files: '**/build/test-results/**/*.xml'
112+
large_files: true

0 commit comments

Comments
 (0)