Skip to content

Commit 0699f35

Browse files
arunpandianparunpandianp
authored
Add post submit test for dataflow streaming with windmill tag encoding v2 (#37603)
1 parent bbaace8 commit 0699f35

3 files changed

Lines changed: 140 additions & 23 deletions

File tree

‎.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"comment": "Modify this file in a trivial way to cause this test suite to run!",
3+
"modification": 1
4+
}

‎.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Java ValidatesRunner Dataflow Streaming TagEncodingV2
19+
20+
on:
21+
schedule:
22+
- cron: '30 4/8 * * *'
23+
pull_request_target:
24+
paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2.json']
25+
workflow_dispatch:
26+
27+
# This allows a subsequently queued workflow run to interrupt previous runs
28+
concurrency:
29+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
30+
cancel-in-progress: true
31+
32+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
33+
permissions:
34+
actions: write
35+
pull-requests: write
36+
checks: write
37+
contents: read
38+
deployments: read
39+
id-token: none
40+
issues: write
41+
discussions: read
42+
packages: read
43+
pages: read
44+
repository-projects: read
45+
security-events: read
46+
statuses: read
47+
48+
env:
49+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
50+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
51+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
52+
53+
jobs:
54+
beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2:
55+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
56+
runs-on: [self-hosted, ubuntu-20.04, main]
57+
timeout-minutes: 720
58+
strategy:
59+
matrix:
60+
job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming_TagEncodingV2]
61+
job_phrase: [Run Java Dataflow Streaming TagEncodingV2 ValidatesRunner]
62+
if: |
63+
github.event_name == 'workflow_dispatch' ||
64+
github.event_name == 'pull_request_target' ||
65+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
66+
github.event.comment.body == 'Run Java Dataflow Streaming TagEncodingV2 ValidatesRunner'
67+
steps:
68+
- uses: actions/checkout@v4
69+
- name: Setup repository
70+
uses: ./.github/actions/setup-action
71+
with:
72+
comment_phrase: ${{ matrix.job_phrase }}
73+
github_token: ${{ secrets.GITHUB_TOKEN }}
74+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
75+
- name: Setup environment
76+
uses: ./.github/actions/setup-environment-action
77+
with:
78+
java-version: default
79+
- name: run validatesRunnerStreamingTagEncodingV2 script
80+
uses: ./.github/actions/gradle-command-self-hosted-action
81+
with:
82+
gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerStreamingTagEncodingV2
83+
max-workers: 12
84+
- name: Archive JUnit Test Results
85+
uses: actions/upload-artifact@v4
86+
if: ${{ !success() }}
87+
with:
88+
name: JUnit Test Results
89+
path: "**/build/reports/tests/"
90+
- name: Publish JUnit Test Results
91+
uses: EnricoMi/publish-unit-test-result-action@v2
92+
if: always()
93+
with:
94+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
95+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
96+
files: '**/build/test-results/**/*.xml'
97+
large_files: true

‎runners/google-cloud-dataflow-java/build.gradle‎

Lines changed: 39 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -458,34 +458,50 @@ task validatesRunner {
458458
))
459459
}
460460

461+
def validatesRunnerStreamingConfig = [
462+
pipelineOptions: legacyPipelineOptions + ['--streaming'],
463+
excludedCategories: [
464+
'org.apache.beam.sdk.testing.UsesCommittedMetrics',
465+
'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput',
466+
],
467+
excludedTests: [
468+
// TODO(https://github.com/apache/beam/issues/21472)
469+
'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState',
470+
// GroupIntoBatches.withShardedKey not supported on streaming runner v1
471+
// https://github.com/apache/beam/issues/22592
472+
'org.apache.beam.sdk.transforms.GroupIntoBatchesTest.testWithShardedKeyInGlobalWindow',
473+
474+
// These tests use static state and don't work with remote execution.
475+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle',
476+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful',
477+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement',
478+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful',
479+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup',
480+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful',
481+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle',
482+
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful',
483+
]
484+
]
485+
461486
task validatesRunnerStreaming {
462487
group = "Verification"
463488
description "Validates Dataflow runner forcing streaming mode"
464-
dependsOn(createLegacyWorkerValidatesRunnerTest(
489+
dependsOn(createLegacyWorkerValidatesRunnerTest(validatesRunnerStreamingConfig + [
465490
name: 'validatesRunnerLegacyWorkerTestStreaming',
466-
pipelineOptions: legacyPipelineOptions + ['--streaming'],
467-
excludedCategories: [
468-
'org.apache.beam.sdk.testing.UsesCommittedMetrics',
469-
'org.apache.beam.sdk.testing.UsesRequiresTimeSortedInput',
470-
],
471-
excludedTests: [
472-
// TODO(https://github.com/apache/beam/issues/21472)
473-
'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState',
474-
// GroupIntoBatches.withShardedKey not supported on streaming runner v1
475-
// https://github.com/apache/beam/issues/22592
476-
'org.apache.beam.sdk.transforms.GroupIntoBatchesTest.testWithShardedKeyInGlobalWindow',
491+
]))
492+
}
477493

478-
// These tests use static state and don't work with remote execution.
479-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle',
480-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful',
481-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement',
482-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful',
483-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup',
484-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful',
485-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle',
486-
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful',
487-
]
488-
))
494+
task validatesRunnerStreamingTagEncodingV2 {
495+
group = "Verification"
496+
description "Validates Dataflow runner (legacy) with Tag Encoding V2 experiments"
497+
dependsOn(createLegacyWorkerValidatesRunnerTest(validatesRunnerStreamingConfig + [
498+
name: 'validatesRunnerLegacyWorkerTestStreamingTagEncodingV2',
499+
pipelineOptions: validatesRunnerStreamingConfig.pipelineOptions + [
500+
'--experiments=enable_streaming_engine',
501+
'--experiments=enable_streaming_engine_state_tag_encoding_v2',
502+
'--experiments=streaming_engine_state_tag_encoding_v2_supported'
503+
],
504+
]))
489505
}
490506

491507
def setupXVR = tasks.register("setupXVR") {

0 commit comments

Comments
 (0)