Skip to content

Commit 21e3338

Browse files
sayakpaulsayakpaul
authored
[CI] allow running tests as PR comments through a bot (#13873)
* start * fix * remove planning
1 parent 12fc496 commit 21e3338

2 files changed

Lines changed: 198 additions & 76 deletions

File tree

‎.github/workflows/pr_comment_gpu_tests.yml‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
name: GPU Tests from PR Comment
2+
3+
# Lets maintainers (admin / write access) run GPU tests on a PR by commenting:
4+
# /diffusers-bot pytest <args>
5+
# e.g. `/diffusers-bot pytest tests/models/test_modeling_common.py -k "some_test"`.
6+
7+
8+
on:
9+
issue_comment:
10+
types: [created]
11+
12+
# Default to read-only; jobs that comment opt into `pull-requests: write` explicitly.
13+
permissions:
14+
contents: read
15+
16+
concurrency:
17+
# A newer command on the same PR supersedes an in-flight one.
18+
group: diffusers-bot-${{ github.event.issue.number }}
19+
cancel-in-progress: true
20+
21+
env:
22+
DIFFUSERS_IS_CI: yes
23+
OMP_NUM_THREADS: 8
24+
MKL_NUM_THREADS: 8
25+
HF_XET_HIGH_PERFORMANCE: 1
26+
PYTEST_TIMEOUT: 600
27+
# Force version overrides across every `uv pip install`: pin tokenizers and the
28+
# torch/torchvision/torchaudio set baked into the image so `-U` installs can't bump
29+
# torch and break torchvision's C++ ABI. Re-written into the file in the install step.
30+
UV_OVERRIDE: /tmp/uv-overrides.txt
31+
32+
jobs:
33+
gate:
34+
name: Authorize & launch
35+
# Only react to `/diffusers-bot pytest …` comments on open PRs.
36+
if: |
37+
github.event.issue.pull_request &&
38+
github.event.issue.state == 'open' &&
39+
startsWith(github.event.comment.body, '/diffusers-bot pytest')
40+
runs-on: ubuntu-22.04
41+
permissions:
42+
pull-requests: write
43+
outputs:
44+
pytest_args: ${{ steps.parse.outputs.pytest_args }}
45+
comment_id: ${{ steps.comment.outputs.comment_id }}
46+
steps:
47+
- name: Check commenter permission
48+
id: auth
49+
env:
50+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
51+
REPO: ${{ github.repository }}
52+
COMMENTER: ${{ github.event.comment.user.login }}
53+
run: |
54+
PERM=$(gh api "repos/${REPO}/collaborators/${COMMENTER}/permission" --jq '.permission' 2>/dev/null || echo "none")
55+
echo "Commenter @${COMMENTER} has permission: ${PERM}"
56+
if [[ "$PERM" == "admin" || "$PERM" == "write" ]]; then
57+
echo "authorized=true" >> "$GITHUB_OUTPUT"
58+
else
59+
echo "authorized=false" >> "$GITHUB_OUTPUT"
60+
fi
61+
62+
- name: Reject unauthorized commenter
63+
if: steps.auth.outputs.authorized != 'true'
64+
env:
65+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
66+
REPO: ${{ github.repository }}
67+
PR: ${{ github.event.issue.number }}
68+
COMMENTER: ${{ github.event.comment.user.login }}
69+
run: |
70+
gh api -X POST "repos/${REPO}/issues/${PR}/comments" \
71+
-f body="🚫 Sorry @${COMMENTER}, you're not authorized to run \`/diffusers-bot\`. Only maintainers with write or admin access can trigger GPU tests." >/dev/null
72+
echo "::error::Only maintainers with write/admin access can run /diffusers-bot."
73+
exit 1
74+
75+
- name: Acknowledge with 👀
76+
env:
77+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
78+
REPO: ${{ github.repository }}
79+
COMMENT_ID: ${{ github.event.comment.id }}
80+
run: |
81+
gh api -X POST "repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -f content="eyes" >/dev/null
82+
83+
- name: Parse pytest args
84+
id: parse
85+
env:
86+
COMMENT_BODY: ${{ github.event.comment.body }}
87+
run: |
88+
# Use only the first line of the comment, strip the command prefix.
89+
FIRST_LINE=$(printf '%s' "$COMMENT_BODY" | head -n1)
90+
ARGS="${FIRST_LINE#/diffusers-bot pytest}"
91+
# Trim surrounding whitespace/CR.
92+
ARGS="$(printf '%s' "$ARGS" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
93+
echo "pytest_args=${ARGS}" >> "$GITHUB_OUTPUT"
94+
95+
- name: Post "running" comment
96+
id: comment
97+
env:
98+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
99+
REPO: ${{ github.repository }}
100+
PR: ${{ github.event.issue.number }}
101+
COMMENTER: ${{ github.event.comment.user.login }}
102+
PYTEST_ARGS: ${{ steps.parse.outputs.pytest_args }}
103+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
104+
run: |
105+
BODY="⏳ Running \`pytest ${PYTEST_ARGS}\` on a GPU runner — [view logs](${RUN_URL}).
106+
107+
Triggered by @${COMMENTER}."
108+
CID=$(gh api -X POST "repos/${REPO}/issues/${PR}/comments" -f body="$BODY" --jq '.id')
109+
echo "comment_id=${CID}" >> "$GITHUB_OUTPUT"
110+
111+
gpu_tests:
112+
name: Run pytest on GPU
113+
needs: gate
114+
runs-on:
115+
group: aws-g4dn-2xlarge
116+
container:
117+
image: diffusers/diffusers-pytorch-cuda
118+
options: --gpus all --shm-size "16gb" --ipc host
119+
# Least privilege: this job checks out and runs untrusted fork code, so it gets no
120+
# write token. Comment writes happen only in `gate`/`report`.
121+
permissions:
122+
contents: read
123+
defaults:
124+
run:
125+
shell: bash
126+
steps:
127+
- name: Checkout PR head
128+
uses: actions/checkout@v6
129+
with:
130+
# Works for forks too — no fork credentials needed.
131+
ref: refs/pull/${{ github.event.issue.number }}/head
132+
fetch-depth: 2
133+
134+
- name: NVIDIA-SMI
135+
run: nvidia-smi
136+
137+
- name: Install dependencies
138+
run: |
139+
printf 'tokenizers<0.23.0\ntorch==2.10.0\ntorchvision==0.25.0\ntorchaudio==2.10.0\n' > "$UV_OVERRIDE"
140+
uv pip install -e ".[quality,training,test]"
141+
uv pip install peft@git+https://github.com/huggingface/peft.git
142+
uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git
143+
uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git
144+
145+
- name: Environment
146+
run: diffusers-cli env
147+
148+
- name: Run pytest
149+
env:
150+
HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
151+
# https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
152+
CUBLAS_WORKSPACE_CONFIG: :16:8
153+
# Forwarded via env (not interpolated into the script) to avoid breakage on
154+
# quotes/special characters in a legitimate command.
155+
PYTEST_ARGS: ${{ needs.gate.outputs.pytest_args }}
156+
run: |
157+
eval "pytest --make-reports=tests_bot_gpu $PYTEST_ARGS"
158+
159+
- name: Failure short reports
160+
if: ${{ failure() }}
161+
run: |
162+
cat reports/tests_bot_gpu_stats.txt || true
163+
cat reports/tests_bot_gpu_failures_short.txt || true
164+
165+
- name: Test suite reports artifacts
166+
if: ${{ always() }}
167+
uses: actions/upload-artifact@v6
168+
with:
169+
name: bot_gpu_test_reports
170+
path: reports
171+
172+
report:
173+
name: Report status
174+
needs: [gate, gpu_tests]
175+
# Always run so the comment is updated on success, failure, or cancellation —
176+
# but only if `gate` actually posted a comment to update.
177+
if: ${{ always() && needs.gate.outputs.comment_id != '' }}
178+
runs-on: ubuntu-22.04
179+
permissions:
180+
pull-requests: write
181+
steps:
182+
- name: Update comment with final status
183+
env:
184+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
185+
REPO: ${{ github.repository }}
186+
CID: ${{ needs.gate.outputs.comment_id }}
187+
RESULT: ${{ needs.gpu_tests.result }}
188+
PYTEST_ARGS: ${{ needs.gate.outputs.pytest_args }}
189+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
190+
run: |
191+
case "$RESULT" in
192+
success) EMOJI="✅"; MSG="passed";;
193+
failure) EMOJI="❌"; MSG="failed";;
194+
cancelled) EMOJI="⚠️"; MSG="was cancelled";;
195+
*) EMOJI="⚠️"; MSG="did not run (${RESULT})";;
196+
esac
197+
BODY="${EMOJI} \`pytest ${PYTEST_ARGS}\` ${MSG} on GPU — [view logs](${RUN_URL})."
198+
gh api -X PATCH "repos/${REPO}/issues/comments/${CID}" -f body="$BODY"

‎.github/workflows/run_tests_from_a_pr.yml‎

Lines changed: 0 additions & 76 deletions
This file was deleted.

0 commit comments

Comments
 (0)