Skip to content

Commit 956b085

Browse files
MintsIncMintsInc
committed
Convert python testing workflow to reusable workflow
This PR is part of a larger initiative to standardize CI workflows across all Datadog API client repositories. By converting our Python testing workflow to a reusable workflow, we can: - **Enable centralized CI management**: The datadog-api-spec repo will be able to use the same testing workflow - **Prepare for MergeQueue**: Centralizing the CI in the datadog-api-spec repo is necessary to enable the MergeQueue ## Changes ### New Files - **`.github/workflows/reusable-python-test.yml`**: A reusable workflow that contains the same logic as the original test workflow ### Modified Files - **`.github/workflows/test.yml`**: Simplified to call the reusable workflow ### Key Design Decisions - **Behavior preservation**: Every aspect of the original workflow is maintained through input parameters - **Configurability**: The reusable workflow accepts inputs for all major parameters (Python versions, platforms, matrix exclusions) - **Environment variables**: Git author information is inherited from the caller workflow
1 parent cc042b6 commit 956b085

File tree

7 files changed

+439
-114
lines changed

7 files changed

+439
-114
lines changed

‎.github/workflows/reusable-ci.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: Reusable Complete CI Workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
target-branch:
7+
description: 'Branch to checkout and test (defaults to the calling branch)'
8+
required: false
9+
type: string
10+
default: ''
11+
enable-commit-changes:
12+
description: 'Whether to commit and push pre-commit fixes'
13+
required: false
14+
type: boolean
15+
default: true
16+
enable-status-reporting:
17+
description: 'Whether to enable status reporting'
18+
required: false
19+
type: boolean
20+
default: true
21+
status-context:
22+
description: 'Context for status checks'
23+
required: false
24+
type: string
25+
default: 'master/unit'
26+
target-repo:
27+
description: 'Repository to post status to'
28+
required: false
29+
type: string
30+
default: 'datadog-api-spec'
31+
python-versions:
32+
description: 'JSON array of Python versions to test against'
33+
required: false
34+
type: string
35+
default: '["3.8", "3.12"]'
36+
platforms:
37+
description: 'JSON array of platforms to run tests on'
38+
required: false
39+
type: string
40+
default: '["ubuntu-22.04", "ubuntu-latest", "macos-latest"]'
41+
matrix-exclude:
42+
description: 'JSON array of matrix combinations to exclude'
43+
required: false
44+
type: string
45+
default: '[{"platform": "macos-latest", "python-version": "3.8"}, {"platform": "ubuntu-latest", "python-version": "3.8"}, {"platform": "ubuntu-22.04", "python-version": "3.12"}]'
46+
secrets:
47+
PIPELINE_GITHUB_APP_ID:
48+
required: false
49+
PIPELINE_GITHUB_APP_PRIVATE_KEY:
50+
required: false
51+
# Integration test secrets
52+
DD_API_KEY:
53+
required: false
54+
DD_CLIENT_API_KEY:
55+
required: false
56+
DD_CLIENT_APP_KEY:
57+
required: false
58+
SLEEP_AFTER_REQUEST:
59+
required: false
60+
61+
jobs:
62+
pre-commit:
63+
uses: ./.github/workflows/reusable-pre-commit.yml
64+
with:
65+
target-branch: ${{ inputs.target-branch }}
66+
enable-commit-changes: ${{ inputs.enable-commit-changes }}
67+
secrets:
68+
PIPELINE_GITHUB_APP_ID: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
69+
PIPELINE_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
70+
71+
test:
72+
uses: ./.github/workflows/reusable-python-test.yml
73+
with:
74+
target-branch: ${{ inputs.target-branch }}
75+
python-versions: ${{ inputs.python-versions }}
76+
platforms: ${{ inputs.platforms }}
77+
matrix-exclude: ${{ inputs.matrix-exclude }}
78+
enable-status-reporting: false # We handle reporting in the main report job
79+
status-context: ${{ inputs.status-context }}
80+
secrets:
81+
PIPELINE_GITHUB_APP_ID: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
82+
PIPELINE_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
83+
84+
examples:
85+
uses: ./.github/workflows/reusable-examples.yml
86+
with:
87+
target-branch: ${{ inputs.target-branch }}
88+
89+
integration:
90+
uses: ./.github/workflows/reusable-integration-test.yml
91+
with:
92+
target-branch: ${{ inputs.target-branch }}
93+
enable-status-reporting: false # We handle reporting in the main report job
94+
status-context: 'integration'
95+
target-repo: ${{ inputs.target-repo }}
96+
secrets:
97+
PIPELINE_GITHUB_APP_ID: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
98+
PIPELINE_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
99+
DD_API_KEY: ${{ secrets.DD_API_KEY }}
100+
DD_CLIENT_API_KEY: ${{ secrets.DD_CLIENT_API_KEY }}
101+
DD_CLIENT_APP_KEY: ${{ secrets.DD_CLIENT_APP_KEY }}
102+
SLEEP_AFTER_REQUEST: ${{ secrets.SLEEP_AFTER_REQUEST }}
103+
104+
report:
105+
uses: ./.github/workflows/reusable-report.yml
106+
needs:
107+
- test
108+
- examples
109+
- integration
110+
if: always()
111+
with:
112+
test-result: ${{ needs.test.result }}
113+
examples-result: ${{ needs.examples.result }}
114+
integration-result: ${{ needs.integration.result }}
115+
context: ${{ inputs.status-context }}
116+
target-repo: ${{ inputs.target-repo }}
117+
enable-status-reporting: ${{ inputs.enable-status-reporting }}
118+
secrets:
119+
PIPELINE_GITHUB_APP_ID: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
120+
PIPELINE_GITHUB_APP_PRIVATE_KEY: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}

‎.github/workflows/reusable-examples.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Reusable Examples Workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
target-branch:
7+
description: 'Branch to checkout and test (defaults to the calling branch)'
8+
required: false
9+
type: string
10+
default: ''
11+
12+
jobs:
13+
examples:
14+
runs-on: ubuntu-latest
15+
if: >
16+
(github.event.pull_request.draft == false &&
17+
!contains(github.event.pull_request.labels.*.name, 'ci/skip') &&
18+
!contains(github.event.pull_request.head.ref, 'datadog-api-spec/test/')) ||
19+
github.event_name == 'schedule'
20+
steps:
21+
- uses: actions/checkout@v3
22+
with:
23+
repository: DataDog/datadog-api-client-python
24+
ref: ${{ inputs.target-branch || github.ref }}
25+
- name: Install Python
26+
uses: actions/setup-python@v4
27+
with:
28+
python-version: "3.12"
29+
cache: "pip"
30+
- name: Upgrade pip
31+
run: |
32+
python -m pip install --upgrade pip
33+
pip install --upgrade wheel setuptools build
34+
- name: Install
35+
run: pip install --disable-pip-version-check pyflakes
36+
- name: Check examples
37+
run: ./check-examples.sh
38+
shell: bash

.github/workflows/test_integration.yml renamed to .github/workflows/reusable-integration-test.yml

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Run Integration Tests
1+
name: Reusable Integration Test Workflow
22

33
permissions:
44
contents: read
@@ -16,6 +16,41 @@ on:
1616
- master
1717
schedule:
1818
- cron: "0 1 * * *"
19+
workflow_call:
20+
inputs:
21+
target-branch:
22+
description: 'Branch to checkout and test (defaults to the calling branch)'
23+
required: false
24+
type: string
25+
default: ''
26+
enable-status-reporting:
27+
description: 'Whether to post status checks to datadog-api-spec repo'
28+
required: false
29+
type: boolean
30+
default: false
31+
status-context:
32+
description: 'Context for status checks'
33+
required: false
34+
type: string
35+
default: 'integration'
36+
target-repo:
37+
description: 'Repository to post status to'
38+
required: false
39+
type: string
40+
default: 'datadog-api-spec'
41+
secrets:
42+
PIPELINE_GITHUB_APP_ID:
43+
required: false
44+
PIPELINE_GITHUB_APP_PRIVATE_KEY:
45+
required: false
46+
DD_API_KEY:
47+
required: true
48+
DD_CLIENT_API_KEY:
49+
required: true
50+
DD_CLIENT_APP_KEY:
51+
required: true
52+
SLEEP_AFTER_REQUEST:
53+
required: false
1954

2055
concurrency:
2156
group: integration-${{ github.head_ref }}
@@ -48,17 +83,20 @@ jobs:
4883
with:
4984
app-id: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
5085
private-key: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
51-
repositories: datadog-api-spec
86+
repositories: ${{ inputs.target-repo || 'datadog-api-spec' }}
5287
- name: Checkout code
5388
uses: actions/checkout@v3
89+
with:
90+
repository: DataDog/datadog-api-client-python
91+
ref: ${{ inputs.target-branch || github.ref }}
5492
- name: Post pending status check
55-
if: github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/')
93+
if: github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/') && (inputs.enable-status-reporting || github.event_name != 'workflow_call')
5694
uses: DataDog/github-actions/post-status-check@v2
5795
with:
5896
github-token: ${{ steps.get_token.outputs.token }}
59-
repo: datadog-api-spec
97+
repo: ${{ inputs.target-repo || 'datadog-api-spec' }}
6098
status: pending
61-
context: integration
99+
context: ${{ inputs.status-context || 'integration' }}
62100
- name: Set up Python 3.12
63101
uses: actions/setup-python@v4
64102
with:
@@ -86,18 +124,18 @@ jobs:
86124
DD_PYTEST_OPERATION_NAME: "test"
87125
DD_TRACE_PROPAGATION_STYLE_INJECT: "datadog"
88126
- name: Post failure status check
89-
if: failure() && github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/')
127+
if: failure() && github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/') && (inputs.enable-status-reporting || github.event_name != 'workflow_call')
90128
uses: DataDog/github-actions/post-status-check@v2
91129
with:
92130
github-token: ${{ steps.get_token.outputs.token }}
93-
repo: datadog-api-spec
131+
repo: ${{ inputs.target-repo || 'datadog-api-spec' }}
94132
status: failure
95-
context: integration
133+
context: ${{ inputs.status-context || 'integration' }}
96134
- name: Post success status check
97-
if: "!failure() && github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/')"
135+
if: "!failure() && github.event_name == 'pull_request' && contains(github.event.pull_request.head.ref, 'datadog-api-spec/generated/') && (inputs.enable-status-reporting || github.event_name != 'workflow_call')"
98136
uses: DataDog/github-actions/post-status-check@v2
99137
with:
100138
github-token: ${{ steps.get_token.outputs.token }}
101-
repo: datadog-api-spec
139+
repo: ${{ inputs.target-repo || 'datadog-api-spec' }}
102140
status: success
103-
context: integration
141+
context: ${{ inputs.status-context || 'integration' }}

‎.github/workflows/reusable-pre-commit.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Reusable Pre-commit Workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
target-branch:
7+
description: 'Branch to checkout and test (defaults to the calling branch)'
8+
required: false
9+
type: string
10+
default: ''
11+
enable-commit-changes:
12+
description: 'Whether to commit and push pre-commit fixes'
13+
required: false
14+
type: boolean
15+
default: true
16+
secrets:
17+
PIPELINE_GITHUB_APP_ID:
18+
required: false
19+
PIPELINE_GITHUB_APP_PRIVATE_KEY:
20+
required: false
21+
22+
env:
23+
GIT_AUTHOR_EMAIL: "packages@datadoghq.com"
24+
GIT_AUTHOR_NAME: "ci.datadog-api-spec"
25+
26+
jobs:
27+
pre-commit:
28+
runs-on: ubuntu-latest
29+
if: >
30+
(github.event.pull_request.draft == false &&
31+
!contains(github.event.pull_request.labels.*.name, 'ci/skip') &&
32+
!contains(github.event.pull_request.head.ref, 'datadog-api-spec/test/')) ||
33+
github.event_name == 'schedule'
34+
steps:
35+
- name: Get GitHub App token
36+
id: get_token
37+
if: inputs.enable-commit-changes
38+
uses: actions/create-github-app-token@v1
39+
with:
40+
app-id: ${{ secrets.PIPELINE_GITHUB_APP_ID }}
41+
private-key: ${{ secrets.PIPELINE_GITHUB_APP_PRIVATE_KEY }}
42+
- uses: actions/checkout@v3
43+
with:
44+
fetch-depth: 0
45+
repository: DataDog/datadog-api-client-python
46+
ref: ${{ inputs.target-branch || github.event.pull_request.head.sha || github.ref }}
47+
token: ${{ inputs.enable-commit-changes && steps.get_token.outputs.token || github.token }}
48+
- uses: actions/setup-python@v4
49+
with:
50+
python-version: '3.11'
51+
- name: Install pre-commit
52+
run: python -m pip install pre-commit
53+
- name: set PY
54+
run: echo "PY=$(python -c 'import platform;print(platform.python_version())')" >> $GITHUB_ENV
55+
- uses: actions/cache@v3
56+
with:
57+
path: ~/.cache/pre-commit
58+
key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
59+
- id: pre_commit
60+
name: Run pre-commit
61+
if: github.event.action != 'closed' && github.event.pull_request.merged != true
62+
run: |
63+
pre-commit run --from-ref "${FROM_REF}" --to-ref "${TO_REF}" --show-diff-on-failure --color=always
64+
env:
65+
FROM_REF: ${{ github.event.pull_request.base.sha }}
66+
TO_REF: ${{ github.event.pull_request.head.sha }}
67+
- name: Commit changes
68+
if: ${{ failure() && inputs.enable-commit-changes }}
69+
run: |-
70+
git add -A
71+
git config user.name "${GIT_AUTHOR_NAME}"
72+
git config user.email "${GIT_AUTHOR_EMAIL}"
73+
git commit -m "pre-commit fixes"
74+
git push origin "HEAD:${HEAD_REF}"
75+
exit 1
76+
env:
77+
HEAD_REF: ${{ github.event.pull_request.head.ref }}
78+
- id: pre_commit_schedule
79+
name: Run pre-commit in schedule
80+
if: github.event_name == 'schedule'
81+
run: |
82+
pre-commit run --all-files --show-diff-on-failure --color=always

0 commit comments

Comments
 (0)