Skip to content

Commit b10546c

Browse files
Amar3ttoAmar3tto
authored
Add Load Tests SideInput Flink Batch Go workflow (#28662)
* Add Load Tests SideInput Flink Batch Go workflow * Refactoring * Refactoring
1 parent 0a6b29d commit b10546c

3 files changed

Lines changed: 168 additions & 0 deletions

File tree

‎.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: LoadTests Go SideInput Flink Batch
17+
18+
on:
19+
issue_comment:
20+
types: [created]
21+
schedule:
22+
- cron: '40 23 * * *'
23+
workflow_dispatch:
24+
25+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
26+
permissions:
27+
actions: write
28+
pull-requests: read
29+
checks: read
30+
contents: read
31+
deployments: read
32+
id-token: none
33+
issues: read
34+
discussions: read
35+
packages: read
36+
pages: read
37+
repository-projects: read
38+
security-events: read
39+
statuses: read
40+
41+
# This allows a subsequently queued workflow run to interrupt previous runs
42+
concurrency:
43+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login }}'
44+
cancel-in-progress: true
45+
46+
env:
47+
GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
48+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
49+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
50+
GCLOUD_ZONE: us-central1-a
51+
CLUSTER_NAME: beam-loadtests-go-sideinput-flink-batch-${{ github.run_id }}
52+
GCS_BUCKET: gs://beam-flink-cluster
53+
FLINK_DOWNLOAD_URL: https://archive.apache.org/dist/flink/flink-1.15.0/flink-1.15.0-bin-scala_2.12.tgz
54+
HADOOP_DOWNLOAD_URL: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
55+
FLINK_TASKMANAGER_SLOTS: 1
56+
DETACHED_MODE: true
57+
HARNESS_IMAGES_TO_PULL: gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
58+
JOB_SERVER_IMAGE: gcr.io/apache-beam-testing/beam_portability/beam_flink1.15_job_server:latest
59+
ARTIFACTS_DIR: gs://beam-flink-cluster/beam-loadtests-go-sideinput-flink-batch-${{ github.run_id }}
60+
61+
jobs:
62+
beam_LoadTests_Go_SideInput_Flink_Batch:
63+
if: |
64+
github.event_name == 'workflow_dispatch' ||
65+
github.event_name == 'schedule' ||
66+
github.event.comment.body == 'Run Load Tests Go SideInput Flink Batch'
67+
runs-on: [self-hosted, ubuntu-20.04, main]
68+
timeout-minutes: 720
69+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
70+
strategy:
71+
matrix:
72+
job_name: ["beam_LoadTests_Go_SideInput_Flink_Batch"]
73+
job_phrase: ["Run Load Tests Go SideInput Flink Batch"]
74+
steps:
75+
- uses: actions/checkout@v3
76+
- name: Setup repository
77+
uses: ./.github/actions/setup-action
78+
with:
79+
comment_phrase: ${{ matrix.job_phrase }}
80+
github_token: ${{ secrets.GITHUB_TOKEN }}
81+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
82+
- name: Prepare test arguments
83+
uses: ./.github/actions/test-arguments-action
84+
with:
85+
test-type: load
86+
test-language: go
87+
argument-file-paths: |
88+
${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt
89+
${{ github.workspace }}/.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt
90+
arguments: |
91+
--job_name=load-tests-go-flink-batch-sideinput-$(date '+%m%d%H%M%S' --utc)
92+
- name: Start Flink with parallelism 10
93+
env:
94+
FLINK_NUM_WORKERS: 10
95+
run: |
96+
cd ${{ github.workspace }}/.test-infra/dataproc; ./flink_cluster.sh create
97+
# The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
98+
- name: run SideInput Flink Batch Go Load Test 1 (first iterable)
99+
timeout-minutes: 240
100+
uses: ./.github/actions/gradle-command-self-hosted-action
101+
with:
102+
gradle-command: :sdks:go:test:load:run
103+
arguments: |
104+
-PloadTest.mainClass=sideinput \
105+
-Prunner=FlinkRunner \
106+
'-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Flink_Batch_test_arguments_1 }}' \
107+
- name: run SideInput Flink Batch Go Load Test 2 (iterable)
108+
timeout-minutes: 240
109+
uses: ./.github/actions/gradle-command-self-hosted-action
110+
with:
111+
gradle-command: :sdks:go:test:load:run
112+
arguments: |
113+
-PloadTest.mainClass=sideinput \
114+
-Prunner=FlinkRunner \
115+
'-PloadTest.args=${{ env.beam_LoadTests_Go_SideInput_Flink_Batch_test_arguments_2 }}'
116+
- name: Teardown Flink
117+
if: always()
118+
run: |
119+
${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete

‎.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_First_Iterable.txt‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--influx_namespace=flink
18+
--influx_measurement=go_batch_sideinput_1
19+
--input_options=''{\"num_records\":400000,\"key_size\":100,\"value_size\":900}''
20+
--access_percentage=1
21+
--parallelism=10
22+
--endpoint=localhost:8099
23+
--environment_type=DOCKER
24+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
25+
--runner=FlinkRunner

‎.github/workflows/load-tests-job-configs/go_SideInput_Flink_Batch_Iterable.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
--influx_namespace=flink
18+
--influx_measurement=go_batch_sideinput_2
19+
--input_options=''{\"num_records\":400000,\"key_size\":100,\"value_size\":900}''
20+
--parallelism=10
21+
--endpoint=localhost:8099
22+
--environment_type=DOCKER
23+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
24+
--runner=FlinkRunner

0 commit comments

Comments
 (0)