Skip to content

Commit e21c9dc

Browse files
cquil11cquil11
committed
debug 8
1 parent dfd4e37 commit e21c9dc

1 file changed

Lines changed: 127 additions & 123 deletions

File tree

‎.github/workflows/label-validation.yml‎

Lines changed: 127 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,134 @@
11
name: PR Label Validation
22
run-name: "Validate PR #${{ github.event.pull_request.number }}"
33

4+
concurrency:
5+
group: "PR#${{ github.event.pull_request.number }}"
6+
cancel-in-progress: true
7+
48
on:
5-
pull_request:
6-
types: [labeled, synchronize]
7-
branches:
8-
- main
9+
pull_request:
10+
types: [labeled, synchronize]
11+
branches:
12+
- main
913

1014
jobs:
11-
get-jobs:
12-
runs-on: ubuntu-latest
13-
outputs:
14-
search-space-config: ${{ steps.get-jobs.outputs.search-space-config }}
15-
steps:
16-
- name: Checkout code
17-
uses: actions/checkout@v4
18-
19-
- id: get-jobs
20-
shell: python
21-
run: |
22-
import json
23-
import subprocess
24-
import re
25-
import os
26-
27-
# Get matching labels
28-
labels = json.loads(r'''${{ toJson(github.event.pull_request.labels) }}''')
29-
pattern = r'^([^_]+)_([^_]+)$'
30-
31-
matching = []
32-
for label in labels:
33-
match = re.match(pattern, label['name'])
34-
if match:
35-
matching.append({'runner-type': match.group(1), 'model-prefix': match.group(2)})
36-
print(f"Matched label: {label['name']}")
37-
38-
if not matching:
39-
print("No matching labels found")
40-
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
41-
f.write('search-space-config=[]\n')
42-
exit(0)
43-
44-
# Generate configs for all matching labels
45-
subprocess.run(['pip', 'install', 'pydantic'], check=True)
46-
47-
all_configs = []
48-
for label in matching:
49-
result = subprocess.run([
50-
'python3', f"{os.environ['GITHUB_WORKSPACE']}/utils/matrix-logic/generate_sweep_configs.py",
51-
'full-sweep',
52-
'--runner-type', label['runner-type'],
53-
'--model-prefix', label['model-prefix'],
54-
'--seq-lens', '1k1k',
55-
'--test-mode',
56-
'--config-files',
57-
f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/nvidia-master.yaml",
58-
f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/amd-master.yaml",
59-
'--runner-config', f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/runners.yaml"
60-
], capture_output=True, text=True)
61-
62-
if result.returncode != 0:
63-
print(f"Error generating configs:")
64-
print(f"STDOUT: {result.stdout}")
65-
print(f"STDERR: {result.stderr}")
66-
exit(1)
67-
68-
all_configs.extend(json.loads(result.stdout))
69-
70-
print(f"Total configs: {len(all_configs)}")
71-
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
72-
f.write(f'search-space-config={json.dumps(all_configs)}\n')
73-
74-
validate:
75-
needs: get-jobs
76-
# Prolly unnecessary
77-
if: ${{ needs.get-jobs.outputs.search-space-config != '[]' }}
78-
uses: ./.github/workflows/benchmark-tmpl.yml
79-
strategy:
80-
fail-fast: false
81-
matrix:
82-
config: ${{ fromJson(needs.get-jobs.outputs.search-space-config) }}
83-
secrets: inherit
15+
get-jobs:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
search-space-config: ${{ steps.get-jobs.outputs.search-space-config }}
19+
steps:
20+
- name: Checkout code
21+
uses: actions/checkout@v4
22+
23+
- id: get-jobs
24+
shell: python
25+
run: |
26+
import json
27+
import subprocess
28+
import re
29+
import os
30+
31+
# Get matching labels
32+
labels = json.loads(r'''${{ toJson(github.event.pull_request.labels) }}''')
33+
pattern = r'^([^_]+)_([^_]+)$'
34+
35+
matching = []
36+
for label in labels:
37+
match = re.match(pattern, label['name'])
38+
if match:
39+
matching.append({'runner-type': match.group(1), 'model-prefix': match.group(2)})
40+
print(f"Matched label: {label['name']}")
41+
42+
if not matching:
43+
print("No matching labels found")
44+
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
45+
f.write('search-space-config=[]\n')
46+
exit(0)
47+
48+
# Generate configs for all matching labels
49+
subprocess.run(['pip', 'install', 'pydantic'], check=True)
50+
51+
all_configs = []
52+
for label in matching:
53+
result = subprocess.run([
54+
'python3', f"{os.environ['GITHUB_WORKSPACE']}/utils/matrix-logic/generate_sweep_configs.py",
55+
'full-sweep',
56+
'--runner-type', label['runner-type'],
57+
'--model-prefix', label['model-prefix'],
58+
'--seq-lens', '1k1k',
59+
'--test-mode',
60+
'--config-files',
61+
f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/nvidia-master.yaml",
62+
f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/amd-master.yaml",
63+
'--runner-config', f"{os.environ['GITHUB_WORKSPACE']}/.github/configs/runners.yaml"
64+
], capture_output=True, text=True)
65+
66+
if result.returncode != 0:
67+
print(f"Error generating configs:")
68+
print(f"STDOUT: {result.stdout}")
69+
print(f"STDERR: {result.stderr}")
70+
exit(1)
71+
72+
all_configs.extend(json.loads(result.stdout))
73+
74+
print(f"Total configs: {len(all_configs)}")
75+
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
76+
f.write(f'search-space-config={json.dumps(all_configs)}\n')
77+
78+
validate:
79+
needs: get-jobs
80+
# Prolly unnecessary
81+
if: ${{ needs.get-jobs.outputs.search-space-config != '[]' }}
82+
uses: ./.github/workflows/benchmark-tmpl.yml
83+
strategy:
84+
fail-fast: false
85+
matrix:
86+
config: ${{ fromJson(needs.get-jobs.outputs.search-space-config) }}
87+
secrets: inherit
88+
with:
89+
exp-name: ${{ matrix.config.exp-name }}
90+
isl: ${{ matrix.config.isl }}
91+
osl: ${{ matrix.config.osl }}
92+
max-model-len: ${{ matrix.config.max-model-len }}
93+
runner: ${{ matrix.config.runner }}
94+
image: ${{ matrix.config.image }}
95+
model: ${{ matrix.config.model }}
96+
framework: ${{ matrix.config.framework }}
97+
precision: ${{ matrix.config.precision }}
98+
tp: ${{ matrix.config.tp }}
99+
ep: ${{ matrix.config.ep }}
100+
dp-attn: ${{ matrix.config.dp-attn }}
101+
conc: ${{ matrix.config.conc }}
102+
103+
calc-success-rate:
104+
needs: validate
105+
if: ${{ always() }}
106+
runs-on: ubuntu-latest
107+
108+
env:
109+
RESULTS_DIR: "results/"
110+
STATS_FILENAME: "run_stats"
111+
GITHUB_TOKEN: ${{ secrets.REPO_PAT }}
112+
113+
steps:
114+
- uses: actions/checkout@v3
115+
with:
116+
token: ${{ secrets.REPO_PAT }}
117+
fetch-depth: 0
118+
119+
- name: Download results artifacts
120+
uses: actions/download-artifact@v4
121+
with:
122+
path: ${{ env.RESULTS_DIR }}
123+
pattern: results_*
124+
125+
- name: Install python dependencies
126+
run: pip install PyGithub
127+
128+
- name: Calculate success rate
129+
run: python3 utils/calc_success_rate.py $STATS_FILENAME
130+
131+
- uses: actions/upload-artifact@v4
84132
with:
85-
exp-name: ${{ matrix.config.exp-name }}
86-
isl: ${{ matrix.config.isl }}
87-
osl: ${{ matrix.config.osl }}
88-
max-model-len: ${{ matrix.config.max-model-len }}
89-
runner: ${{ matrix.config.runner }}
90-
image: ${{ matrix.config.image }}
91-
model: ${{ matrix.config.model }}
92-
framework: ${{ matrix.config.framework }}
93-
precision: ${{ matrix.config.precision }}
94-
tp: ${{ matrix.config.tp }}
95-
ep: ${{ matrix.config.ep }}
96-
dp-attn: ${{ matrix.config.dp-attn }}
97-
conc: ${{ matrix.config.conc }}
98-
99-
calc-success-rate:
100-
needs: validate
101-
if: ${{ always() }}
102-
runs-on: ubuntu-latest
103-
104-
env:
105-
RESULTS_DIR: "results/"
106-
STATS_FILENAME: "run_stats"
107-
GITHUB_TOKEN: ${{ secrets.REPO_PAT }}
108-
109-
steps:
110-
- uses: actions/checkout@v3
111-
with:
112-
token: ${{ secrets.REPO_PAT }}
113-
fetch-depth: 0
114-
115-
- name: Download results artifacts
116-
uses: actions/download-artifact@v4
117-
with:
118-
path: ${{ env.RESULTS_DIR }}
119-
pattern: results_*
120-
121-
- name: Install python dependencies
122-
run: pip install PyGithub
123-
124-
- name: Calculate success rate
125-
run: python3 utils/calc_success_rate.py $STATS_FILENAME
126-
127-
- uses: actions/upload-artifact@v4
128-
with:
129-
name: "run-stats"
130-
path: ${{ env.STATS_FILENAME }}.json
133+
name: "run-stats"
134+
path: ${{ env.STATS_FILENAME }}.json

0 commit comments

Comments
 (0)