Skip to content

Commit 0fd3787

Browse files
OseltamivirOseltamivir
committed
Initial commit, for #304 
1 parent c7e4b1d commit 0fd3787

2 files changed

Lines changed: 222 additions & 0 deletions

File tree

‎.github/workflows/pr-comment-sweep.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: PR Comment Sweep
2+
3+
on:
4+
issue_comment:
5+
types: [created]
6+
7+
concurrency:
8+
group: "PR-SWEEP-${{ github.event.issue.number }}"
9+
cancel-in-progress: true
10+
11+
permissions:
12+
contents: read
13+
14+
jobs:
15+
parse:
16+
# Run only for PR comments, from trusted authors, starting with /sweep
17+
if: >-
18+
${{ github.event.issue.pull_request &&
19+
startsWith(github.event.comment.body, '/sweep') &&
20+
contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association) }}
21+
runs-on: ubuntu-latest
22+
outputs:
23+
pr-number: ${{ steps.parse.outputs.pr-number }}
24+
generator-args: ${{ steps.parse.outputs.generator-args }}
25+
steps:
26+
- name: Derive PR number and parse command
27+
id: parse
28+
shell: bash
29+
env:
30+
BODY: ${{ github.event.comment.body }}
31+
PR_NUMBER: ${{ github.event.issue.number }}
32+
run: |
33+
set -euo pipefail
34+
35+
# Extract first line starting with /sweep
36+
cmd_line=$(printf "%s" "$BODY" | awk '/^\/sweep/{print; exit}')
37+
if [[ -z "$cmd_line" ]]; then
38+
echo "No /sweep command found in comment" >&2
39+
exit 1
40+
fi
41+
cmd_args=${cmd_line#* /sweep}
42+
# Handle case when it's exactly '/sweep' (no args)
43+
if [[ "$cmd_line" == "/sweep" ]]; then
44+
cmd_args=""
45+
else
46+
cmd_args=${cmd_line#/sweep}
47+
fi
48+
cmd_args=$(echo "$cmd_args" | xargs || true)
49+
50+
echo "Command args: $cmd_args"
51+
52+
echo "generator-args=$cmd_args" >> "$GITHUB_OUTPUT"
53+
echo "pr-number=$PR_NUMBER" >> "$GITHUB_OUTPUT"
54+
55+
execute:
56+
needs: parse
57+
uses: ./.github/workflows/sweep-executor.yml
58+
secrets: inherit
59+
with:
60+
pr-number: ${{ needs.parse.outputs.pr-number }}
61+
generator-args: ${{ needs.parse.outputs.generator-args }}
62+
63+
note-ignored:
64+
# Inform when comment doesn't meet criteria (non-PR or not authorized)
65+
if: ${{ !github.event.issue.pull_request ||
66+
!startsWith(github.event.comment.body, '/sweep') ||
67+
!contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association) }}
68+
runs-on: ubuntu-latest
69+
steps:
70+
- run: |
71+
echo "Comment ignored. Either not on a PR, not a /sweep command, or author not authorized (OWNER/MEMBER/COLLABORATOR required)."

‎.github/workflows/sweep-executor.yml‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
name: Template - Sweep Executor
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
pr-number:
7+
required: true
8+
type: string
9+
generator-args:
10+
required: true
11+
type: string
12+
13+
permissions:
14+
contents: read
15+
16+
jobs:
17+
generate:
18+
runs-on: ubuntu-latest
19+
outputs:
20+
search-space-config: ${{ steps.generate.outputs.search-space-config }}
21+
is-multinode: ${{ steps.detect.outputs.is-multinode }}
22+
steps:
23+
- name: Checkout PR head
24+
uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
25+
with:
26+
ref: refs/pull/${{ inputs.pr-number }}/head
27+
28+
- name: Detect node type from args
29+
id: detect
30+
run: |
31+
set -euo pipefail
32+
ARGS=${{ inputs.generator-args }}
33+
if [[ "$ARGS" == *"--multi-node"* ]]; then
34+
echo "is-multinode=true" >> "$GITHUB_OUTPUT"
35+
else
36+
echo "is-multinode=false" >> "$GITHUB_OUTPUT"
37+
fi
38+
39+
- name: Generate sweep configs
40+
id: generate
41+
shell: python
42+
env:
43+
GITHUB_WORKSPACE: ${{ github.workspace }}
44+
run: |
45+
import json, os, shlex, subprocess, sys
46+
47+
cmd_args = r'''${{ inputs.generator-args }}'''
48+
script = os.path.join(os.environ['GITHUB_WORKSPACE'], 'utils', 'matrix_logic', 'generate_sweep_configs.py')
49+
cfg_amd = os.path.join(os.environ['GITHUB_WORKSPACE'], '.github', 'configs', 'amd-master.yaml')
50+
cfg_nv = os.path.join(os.environ['GITHUB_WORKSPACE'], '.github', 'configs', 'nvidia-master.yaml')
51+
runners = os.path.join(os.environ['GITHUB_WORKSPACE'], '.github', 'configs', 'runners.yaml')
52+
53+
subprocess.run([sys.executable, '-m', 'pip', 'install', 'pydantic'], check=True)
54+
55+
argv = [sys.executable, script]
56+
if cmd_args.strip():
57+
argv += shlex.split(cmd_args)
58+
argv += ['--config-files', cfg_amd, cfg_nv, '--runner-config', runners]
59+
60+
print('Invoking:', ' '.join(shlex.quote(a) for a in argv))
61+
res = subprocess.run(argv, capture_output=True, text=True)
62+
if res.returncode != 0:
63+
print('Generator failed. stdout:\n', res.stdout)
64+
print('stderr:\n', res.stderr, file=sys.stderr)
65+
raise SystemExit(res.returncode)
66+
67+
try:
68+
data = json.loads(res.stdout)
69+
except Exception as e:
70+
print('Failed to parse generator output as JSON:', e, file=sys.stderr)
71+
print('Raw output:\n', res.stdout)
72+
raise
73+
74+
print(f"Generated {len(data)} configs")
75+
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
76+
f.write('search-space-config=' + json.dumps(data) + '\n')
77+
78+
run-single-node:
79+
needs: generate
80+
if: ${{ needs.generate.result == 'success' && needs.generate.outputs.search-space-config != '[]' && needs.generate.outputs.is-multinode == 'false' }}
81+
uses: ./.github/workflows/benchmark-tmpl.yml
82+
name: Sweep (Single-Node)
83+
strategy:
84+
fail-fast: false
85+
matrix:
86+
config: ${{ fromJson(needs.generate.outputs.search-space-config) }}
87+
secrets: inherit
88+
with:
89+
exp-name: ${{ matrix.config.exp-name }}
90+
isl: ${{ matrix.config.isl }}
91+
osl: ${{ matrix.config.osl }}
92+
max-model-len: ${{ matrix.config.max-model-len }}
93+
runner: ${{ matrix.config.runner }}
94+
image: ${{ matrix.config.image }}
95+
model: ${{ matrix.config.model }}
96+
model-prefix: ${{ matrix.config.model-prefix }}
97+
framework: ${{ matrix.config.framework }}
98+
precision: ${{ matrix.config.precision }}
99+
tp: ${{ matrix.config.tp }}
100+
ep: ${{ matrix.config.ep }}
101+
dp-attn: ${{ matrix.config.dp-attn }}
102+
conc: ${{ matrix.config.conc }}
103+
spec-decoding: ${{ matrix.config.spec-decoding }}
104+
disagg: ${{ matrix.config.disagg }}
105+
106+
run-multi-node:
107+
needs: generate
108+
if: ${{ needs.generate.result == 'success' && needs.generate.outputs.search-space-config != '[]' && needs.generate.outputs.is-multinode == 'true' }}
109+
uses: ./.github/workflows/benchmark-multinode-tmpl.yml
110+
name: Sweep (Multi-Node)
111+
strategy:
112+
fail-fast: false
113+
matrix:
114+
config: ${{ fromJson(needs.generate.outputs.search-space-config) }}
115+
secrets: inherit
116+
with:
117+
exp-name: ${{ matrix.config.exp-name }}
118+
isl: ${{ matrix.config.isl }}
119+
osl: ${{ matrix.config.osl }}
120+
max-model-len: ${{ matrix.config.max-model-len }}
121+
runner: ${{ matrix.config.runner }}
122+
image: ${{ matrix.config.image }}
123+
model: ${{ matrix.config.model }}
124+
model-prefix: ${{ matrix.config.model-prefix }}
125+
framework: ${{ matrix.config.framework }}
126+
precision: ${{ matrix.config.precision }}
127+
conc-list: ${{ toJson(matrix.config.conc) }}
128+
spec-decoding: ${{ matrix.config.spec-decoding }}
129+
disagg: ${{ matrix.config.disagg }}
130+
131+
prefill-num-worker: ${{ matrix.config.prefill.num-worker }}
132+
prefill-tp: ${{ matrix.config.prefill.tp }}
133+
prefill-ep: ${{ matrix.config.prefill.ep }}
134+
prefill-dp-attn: ${{ matrix.config.prefill.dp-attn }}
135+
prefill-additional-settings: ${{ toJson(matrix.config.prefill.additional-settings) }}
136+
137+
decode-num-worker: ${{ matrix.config.decode.num-worker }}
138+
decode-tp: ${{ matrix.config.decode.tp }}
139+
decode-ep: ${{ matrix.config.decode.ep }}
140+
decode-dp-attn: ${{ matrix.config.decode.dp-attn }}
141+
decode-additional-settings: ${{ toJson(matrix.config.decode.additional-settings) }}
142+
143+
collect-results:
144+
needs: [run-single-node, run-multi-node]
145+
if: ${{ always() && (needs.run-single-node.result == 'success' || needs.run-multi-node.result == 'success') }}
146+
uses: ./.github/workflows/collect-results.yml
147+
name: Collect Results
148+
secrets: inherit
149+
with:
150+
exp-name: ''
151+

0 commit comments

Comments
 (0)