Skip to content

Commit 94fce18

Browse files
moawnallahmoawnallah
committed
.github: trigger user-defined workflows
1 parent 65a0c4e commit 94fce18

2 files changed

Lines changed: 246 additions & 0 deletions

File tree

‎.github/workflows/beam_PostCommit_Python_Auto.yml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: PostCommit Python Auto
19+
20+
on:
21+
schedule:
22+
- cron: '30 5/6 * * *'
23+
pull_request:
24+
branches: ['master']
25+
workflow_dispatch:
26+
27+
# This allows a subsequently queued workflow run to interrupt previous runs
28+
concurrency:
29+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
30+
cancel-in-progress: true
31+
32+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
33+
permissions:
34+
actions: write
35+
pull-requests: write
36+
checks: write
37+
contents: read
38+
deployments: read
39+
id-token: none
40+
issues: write
41+
discussions: read
42+
packages: read
43+
pages: read
44+
repository-projects: read
45+
security-events: read
46+
statuses: read
47+
48+
env:
49+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
50+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
51+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
52+
ALLOYDB_PASSWORD: ${{ secrets.ALLOYDB_PASSWORD }}
53+
54+
jobs:
55+
beam_PostCommit_Python:
56+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
57+
runs-on:
58+
- [self-hosted, ubuntu-20.04, highmem22]
59+
- ubuntu-latest
60+
timeout-minutes: 240
61+
strategy:
62+
fail-fast: false
63+
matrix:
64+
job_name: [beam_PostCommit_Python]
65+
job_phrase: [Run Python PostCommit]
66+
python_version: ['3.9', '3.10', '3.11', '3.12']
67+
if: |
68+
github.event_name == 'workflow_dispatch' ||
69+
github.event_name == 'pull_request_target' ||
70+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
71+
startswith(github.event.comment.body, 'Run Python PostCommit 3.')
72+
steps:
73+
- uses: actions/checkout@v4
74+
- name: Setup repository
75+
uses: ./.github/actions/setup-action
76+
with:
77+
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
78+
github_token: ${{ secrets.GITHUB_TOKEN }}
79+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
80+
- name: Setup environment
81+
uses: ./.github/actions/setup-environment-action
82+
with:
83+
java-version: |
84+
21
85+
11
86+
python-version: ${{ matrix.python_version }}
87+
- name: Install docker compose
88+
run: |
89+
sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
90+
sudo chmod +x /usr/local/bin/docker-compose
91+
- name: Set PY_VER_CLEAN
92+
id: set_py_ver_clean
93+
run: |
94+
PY_VER=${{ matrix.python_version }}
95+
PY_VER_CLEAN=${PY_VER//.}
96+
echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
97+
- name: run PostCommit Python ${{ matrix.python_version }} script
98+
uses: ./.github/actions/gradle-command-self-hosted-action
99+
with:
100+
gradle-command: :python${{steps.set_py_ver_clean.outputs.py_ver_clean}}PostCommit
101+
arguments: |
102+
-Pjava21Home=$JAVA_HOME_21_X64 \
103+
-PuseWheelDistribution \
104+
# Conditionally add -Pposargs based on the runner type.
105+
-Pposargs="${{
106+
contains(matrix.runner_os, 'self-hosted') &&
107+
'-m ''not require_non_self_hosted''' ||
108+
'-m ''require_non_self_hosted'''
109+
}}" \
110+
-PpythonVersion=${{ matrix.python_version }}
111+
env:
112+
CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
113+
- name: Archive Python Test Results
114+
uses: actions/upload-artifact@v4
115+
if: failure()
116+
with:
117+
name: Python ${{ matrix.python_version }} Test Results
118+
path: '**/pytest*.xml'
119+
- name: Publish Python Test Results
120+
uses: EnricoMi/publish-unit-test-result-action@v2
121+
if: always()
122+
with:
123+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
124+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
125+
files: '**/pytest*.xml'
126+
large_files: true

‎.github/workflows/beam_PreCommit_Python_ML_Auto.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PreCommit Python ML tests with ML deps installed Auto
17+
on:
18+
pull_request:
19+
branches: [ "master", "release-*" ]
20+
issue_comment:
21+
types: [created]
22+
push:
23+
tags: ['v*']
24+
branches: ['master', 'release-*']
25+
paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python_ML.yml"]
26+
schedule:
27+
- cron: '45 2/6 * * *'
28+
workflow_dispatch:
29+
30+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
31+
permissions:
32+
actions: write
33+
pull-requests: write
34+
checks: write
35+
contents: read
36+
deployments: read
37+
id-token: none
38+
issues: write
39+
discussions: read
40+
packages: read
41+
pages: read
42+
repository-projects: read
43+
security-events: read
44+
statuses: read
45+
46+
# This allows a subsequently queued workflow run to interrupt previous runs
47+
concurrency:
48+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
49+
cancel-in-progress: true
50+
51+
env:
52+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
53+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
54+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
55+
ALLOYDB_PASSWORD: ${{ secrets.ALLOYDB_PASSWORD }}
56+
57+
jobs:
58+
beam_PreCommit_Python_ML:
59+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
60+
runs-on:
61+
- [self-hosted, ubuntu-20.04, main]
62+
- ubuntu-latest
63+
timeout-minutes: 180
64+
strategy:
65+
fail-fast: false
66+
matrix:
67+
job_name: ['beam_PreCommit_Python_ML']
68+
job_phrase: ['Run Python_ML PreCommit']
69+
python_version: ['3.9','3.10','3.11','3.12']
70+
if: |
71+
github.event_name == 'push' ||
72+
github.event_name == 'pull_request_target' ||
73+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
74+
github.event_name == 'workflow_dispatch' ||
75+
startsWith(github.event.comment.body, 'Run Python_ML PreCommit')
76+
steps:
77+
- uses: actions/checkout@v4
78+
- name: Setup repository
79+
uses: ./.github/actions/setup-action
80+
with:
81+
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
82+
github_token: ${{ secrets.GITHUB_TOKEN }}
83+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
84+
- name: Setup environment
85+
uses: ./.github/actions/setup-environment-action
86+
with:
87+
java-version: default
88+
python-version: ${{ matrix.python_version }}
89+
- name: Set PY_VER_CLEAN
90+
id: set_py_ver_clean
91+
run: |
92+
PY_VER=${{ matrix.python_version }}
93+
PY_VER_CLEAN=${PY_VER//.}
94+
echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT
95+
- name: Run pythonPreCommit
96+
uses: ./.github/actions/gradle-command-self-hosted-action
97+
with:
98+
gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:testPy${{steps.set_py_ver_clean.outputs.py_ver_clean}}ML
99+
arguments: |
100+
# Conditionally add -Pposargs based on the runner type.
101+
-Pposargs="${{
102+
contains(matrix.runner_os, 'self-hosted') &&
103+
'-m ''not require_non_self_hosted''' ||
104+
'-m ''require_non_self_hosted'''
105+
}}" \
106+
-PpythonVersion=${{ matrix.python_version }}
107+
- name: Archive Python Test Results
108+
uses: actions/upload-artifact@v4
109+
if: failure()
110+
with:
111+
name: Python ${{ matrix.python_version }} Test Results
112+
path: '**/pytest*.xml'
113+
- name: Publish Python Test Results
114+
uses: EnricoMi/publish-unit-test-result-action@v2
115+
if: always()
116+
with:
117+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
118+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
119+
files: '**/pytest*.xml'
120+
large_files: true

0 commit comments

Comments
 (0)