Skip to content

Commit a317020

Browse files
authored
Adds a Dataflow integration test suite for Delta Lake (#38954)
1 parent 92b92a1 commit a317020

5 files changed

Lines changed: 385 additions & 0 deletions

File tree

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3+
"modification": 1
4+
}

.github/workflows/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@ PostCommit Jobs run in a schedule against master branch and generally do not get
330330
| [ PostCommit Java BigQueryEarlyRollout ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml) | N/A |`beam_PostCommit_Java_BigQueryEarlyRollout.json`| [![.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml?query=event%3Aschedule) |
331331
| [ PostCommit Java Dataflow V1 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | N/A |`beam_PostCommit_Java_DataflowV1.json`| [![.github/workflows/beam_PostCommit_Java_DataflowV1.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml?query=event%3Aschedule) |
332332
| [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`beam_PostCommit_Java_DataflowV2.json`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml?query=event%3Aschedule) |
333+
| [ PostCommit Java Delta IO Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Delta_IO_Dataflow.yml) | N/A |`beam_PostCommit_Java_Delta_IO_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Delta_IO_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Delta_IO_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Delta_IO_Dataflow.yml?query=event%3Aschedule) |
333334
| [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | ['8','11','17','21','25'] |`beam_PostCommit_Java_Examples_Dataflow_ARM.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml?query=event%3Aschedule) |
334335
| [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`beam_PostCommit_Java_Examples_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |
335336
| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['8','17','21','25'] |`beam_PostCommit_Java_Examples_Dataflow_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml?query=event%3Aschedule) |
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PostCommit Java Delta IO Dataflow
17+
18+
on:
19+
schedule:
20+
- cron: '30 4/6 * * *'
21+
pull_request_target:
22+
paths: [ 'release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Delta_IO_Dataflow.json' ]
23+
workflow_dispatch:
24+
25+
# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: write
29+
checks: write
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: write
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
44+
cancel-in-progress: true
45+
46+
env:
47+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
51+
jobs:
52+
beam_PostCommit_Java_Delta_IO_Dataflow:
53+
if: |
54+
github.event_name == 'pull_request_target' ||
55+
github.event_name == 'workflow_dispatch' ||
56+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
57+
github.event.comment.body == 'Run PostCommit Java Delta IO Dataflow'
58+
runs-on: [self-hosted, ubuntu-24.04, main]
59+
timeout-minutes: 120
60+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
61+
strategy:
62+
matrix:
63+
job_name: ["beam_PostCommit_Java_Delta_IO_Dataflow"]
64+
job_phrase: ["Run PostCommit Java Delta IO Dataflow"]
65+
steps:
66+
- uses: actions/checkout@v6
67+
- name: Setup repository
68+
uses: ./.github/actions/setup-action
69+
with:
70+
comment_phrase: ${{ matrix.job_phrase }}
71+
github_token: ${{ secrets.GITHUB_TOKEN }}
72+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
73+
- name: Setup environment
74+
uses: ./.github/actions/setup-environment-action
75+
- name: Run PostCommit Java Delta IO Dataflow
76+
uses: ./.github/actions/gradle-command-self-hosted-action
77+
with:
78+
gradle-command: :sdks:java:io:delta:dataflowIntegrationTest --info

sdks/java/io/delta/build.gradle

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
* limitations under the License.
1717
*/
1818

19+
import groovy.json.JsonOutput
20+
1921
plugins { id 'org.apache.beam.module' }
2022
applyJavaNature(
2123
automaticModuleName: 'org.apache.beam.sdk.io.delta',
@@ -68,6 +70,10 @@ dependencies {
6870
testImplementation project(":sdks:java:managed")
6971
testRuntimeOnly "org.yaml:snakeyaml:2.0"
7072
testImplementation project(path: ":runners:direct-java", configuration: "shadow")
73+
testImplementation project(":sdks:java:io:google-cloud-platform")
74+
testImplementation project(":sdks:java:extensions:google-cloud-platform-core")
75+
testImplementation library.java.google_cloud_storage
76+
testRuntimeOnly project(path: ":runners:google-cloud-dataflow-java")
7177
}
7278

7379
configurations.all {
@@ -82,3 +88,35 @@ configurations.all {
8288
resolutionStrategy.force "com.google.cloud.bigdataoss:gcsio:$bigdataoss_gcs_connector_version"
8389
resolutionStrategy.force "com.google.cloud.bigdataoss:util:$bigdataoss_gcs_connector_version"
8490
}
91+
92+
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
93+
def gcpTempLocation = project.findProperty('gcpTempLocation') ?: 'gs://apache-beam-testing-delta-lake/temp'
94+
95+
// Used by Dataflow integration tests.
96+
task dataflowIntegrationTest(type: Test) {
97+
group = "Verification"
98+
evaluationDependsOn(":runners:google-cloud-dataflow-java")
99+
dependsOn ":runners:google-cloud-dataflow-java:buildAndPushDockerJavaContainer"
100+
finalizedBy ":runners:google-cloud-dataflow-java:cleanUpDockerJavaImages"
101+
def dockerJavaImageName = project.project(':runners:google-cloud-dataflow-java').ext.dockerJavaImageName
102+
103+
def args = [
104+
"--runner=DataflowRunner",
105+
"--region=us-central1",
106+
"--project=${gcpProject}",
107+
"--tempLocation=${gcpTempLocation}",
108+
"--tempRoot=${gcpTempLocation}",
109+
"--sdkContainerImage=${dockerJavaImageName}",
110+
"--experiments=use_runner_v2,use_staged_dataflow_worker_jar"
111+
]
112+
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(args)
113+
114+
// Disable Gradle cache: these ITs interact with live service that should always be considered "out of date"
115+
outputs.upToDateWhen { false }
116+
117+
include '**/*IT.class'
118+
119+
maxParallelForks 4
120+
classpath = sourceSets.test.runtimeClasspath
121+
testClassesDirs = sourceSets.test.output.classesDirs
122+
}

0 commit comments

Comments
 (0)