Skip to content

Commit 8e8327d

Browse files
authored
Adds a Github action for running Delta Lake I/O unit tests (#38869)
1 parent 9bcdedd commit 8e8327d

4 files changed

Lines changed: 137 additions & 0 deletions

File tree

.github/autolabeler.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ io: ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/
5252
"aws": ["sdks/java/io/amazon-web-services/**/*", "sdks/java/io/amazon-web-services2/**/*", "sdks/python/apache_beam/io/aws/**/*"]
5353
"cassandra": ["sdks/java/io/cassandra/**/*"]
5454
"clickhouse": ["sdks/java/io/clickhouse/**/*"]
55+
"delta": ["sdks/java/io/delta/**/*"]
5556
"elasticsearch": ["sdks/java/io/elasticsearch/**/*", "sdks/java/io/elasticsearch-tests/**/*"]
5657
"gcp": ["sdks/go/pkg/beam/io/bigqueryio/**/*", "sdks/go/pkg/beam/io/datastoreio/**/*", "sdks/go/pkg/beam/io/pubsubio/**/*", "sdks/java/io/bigquery-io-perf-tests/**/*", "sdks/java/io/google-cloud-platform/**/*", "sdks/python/apache_beam/io/gcp/**/*", "sdks/java/extensions/google-cloud-platform-core/**/*"]
5758
"hadoop": ["sdks/java/io/hadoop-common/**/*", "sdks/java/io/hadoop-file-system/**/*", "sdks/java/io/hadoop-format/**/*"]
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run",
3+
"modification": 1
4+
}

.github/workflows/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
237237
| [ PreCommit Java Clickhouse IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml) | N/A |`Run Java_Clickhouse_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml?query=event%3Aschedule) |
238238
| [ PreCommit Java Csv IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml) | N/A |`Run Java_Csv_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml?query=event%3Aschedule) |
239239
| [ PreCommit Java Datadog IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml) | N/A |`Run Java_Datadog_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml?query=event%3Aschedule) |
240+
| [ PreCommit Java Delta IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml) | N/A |`Run Java_Delta_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml?query=event%3Aschedule) |
240241
| [ PreCommit Java Debezium IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | N/A |`Run Java_Debezium_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml?query=event%3Aschedule) |
241242
| [ PreCommit Java ElasticSearch IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | N/A |`Run Java_ElasticSearch_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml?query=event%3Aschedule) |
242243
| [ PreCommit Java Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java_Examples_Dataflow PreCommit`| [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PreCommit Java Delta IO Direct
17+
18+
on:
19+
push:
20+
tags: ['v*']
21+
branches: ['master', 'release-*']
22+
paths:
23+
- "sdks/java/io/delta/**"
24+
- "sdks/java/managed/**"
25+
- "sdks/java/expansion-service/**"
26+
- "sdks/java/io/expansion-service/**"
27+
- ".github/workflows/beam_PreCommit_Java_Delta_IO_Direct.yml"
28+
pull_request_target:
29+
branches: ['master', 'release-*']
30+
paths:
31+
- "sdks/java/io/delta/**"
32+
- "sdks/java/managed/**"
33+
- "sdks/java/expansion-service/**"
34+
- "sdks/java/io/expansion-service/**"
35+
- 'release/trigger_all_tests.json'
36+
- '.github/trigger_files/beam_PreCommit_Java_Delta_IO_Direct.json'
37+
issue_comment:
38+
types: [created]
39+
schedule:
40+
- cron: '15 2/6 * * *'
41+
workflow_dispatch:
42+
43+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
44+
permissions:
45+
actions: write
46+
pull-requests: write
47+
checks: write
48+
contents: read
49+
deployments: read
50+
id-token: none
51+
issues: write
52+
discussions: read
53+
packages: read
54+
pages: read
55+
repository-projects: read
56+
security-events: read
57+
statuses: read
58+
59+
# This allows a subsequently queued workflow run to interrupt previous runs
60+
concurrency:
61+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
62+
cancel-in-progress: true
63+
64+
env:
65+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
66+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
67+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
68+
69+
jobs:
70+
beam_PreCommit_Java_Delta_IO_Direct:
71+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
72+
strategy:
73+
matrix:
74+
job_name: ["beam_PreCommit_Java_Delta_IO_Direct"]
75+
job_phrase: ["Run Java_Delta_IO_Direct PreCommit"]
76+
timeout-minutes: 60
77+
if: |
78+
github.event_name == 'push' ||
79+
github.event_name == 'pull_request_target' ||
80+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
81+
github.event_name == 'workflow_dispatch' ||
82+
github.event.comment.body == 'Run Java_Delta_IO_Direct PreCommit'
83+
runs-on: [self-hosted, ubuntu-24.04, main]
84+
steps:
85+
- uses: actions/checkout@v6
86+
- name: Setup repository
87+
uses: ./.github/actions/setup-action
88+
with:
89+
comment_phrase: ${{ matrix.job_phrase }}
90+
github_token: ${{ secrets.GITHUB_TOKEN }}
91+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
92+
- name: Setup environment
93+
uses: ./.github/actions/setup-environment-action
94+
with:
95+
java-version: 17
96+
- name: run Delta IO build script
97+
uses: ./.github/actions/gradle-command-self-hosted-action
98+
with:
99+
gradle-command: :sdks:java:io:delta:build
100+
arguments: |
101+
-PdisableSpotlessCheck=true \
102+
-PdisableCheckStyle=true \
103+
-PtestJavaVersion=17 \
104+
-Pjava17Home=$JAVA_HOME_17_X64 \
105+
--info
106+
- name: Archive JUnit Test Results
107+
uses: actions/upload-artifact@v7
108+
if: ${{ !success() }}
109+
with:
110+
name: JUnit Test Results
111+
path: "**/build/reports/tests/"
112+
- name: Publish JUnit Test Results
113+
uses: EnricoMi/publish-unit-test-result-action@v2
114+
if: always()
115+
with:
116+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
117+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
118+
files: '**/build/test-results/**/*.xml'
119+
large_files: true
120+
- name: Archive SpotBugs Results
121+
uses: actions/upload-artifact@v7
122+
if: always()
123+
with:
124+
name: SpotBugs Results
125+
path: '**/build/reports/spotbugs/*.html'
126+
- name: Publish SpotBugs Results
127+
uses: jwgmeligmeyling/spotbugs-github-action@v1.2
128+
if: always()
129+
with:
130+
name: Publish SpotBugs
131+
path: '**/build/reports/spotbugs/*.html'

0 commit comments

Comments
 (0)