Skip to content

Commit fdb94fa

Browse files
cquil11cquil11
committed
adding other isl osl
1 parent 869572a commit fdb94fa

3 files changed

Lines changed: 356 additions & 62 deletions

File tree

‎.github/workflows/1k1k-sweep.yml‎

Lines changed: 66 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
name: "1K/1K Sweep"
22

3+
concurrency:
4+
group: benchmark-lock-1k1k
5+
cancel-in-progress: false
6+
37
on:
48
pull_request:
59
workflow_dispatch:
@@ -28,7 +32,7 @@ jobs:
2832

2933
- id: get-dsr1-configs
3034
run: |
31-
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/get_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix dsr1)
35+
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/get_full_sweep_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix dsr1)
3236
echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
3337
3438
get-gptoss-configs:
@@ -41,56 +45,56 @@ jobs:
4145

4246
- id: get-gptoss-configs
4347
run: |
44-
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/get_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix gptoss)
48+
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/get_full_sweep_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k1k --model-prefix gptoss)
4549
echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
4650
47-
# benchmark-70b:
48-
# needs: get-70b-configs
49-
# uses: ./.github/workflows/benchmark-tmpl.yml
50-
# name: 70b 1k1k
51-
# strategy:
52-
# fail-fast: false
53-
# matrix:
54-
# config: ${{ fromJson(needs.get-70b-configs.outputs.search-space-config) }}
55-
# secrets: inherit
56-
# with:
57-
# exp-name: "70b_1k1k"
58-
# isl: 1024
59-
# osl: 1024
60-
# max-model-len: 2048
61-
# runner: ${{ matrix.config.runner }}
62-
# image: ${{ matrix.config.image }}
63-
# model: ${{ matrix.config.model }}
64-
# framework: ${{ matrix.config.framework }}
65-
# precision: ${{ matrix.config.precision }}
66-
# tp: ${{ matrix.config.tp }}
67-
# ep: ${{ matrix.config.ep || 1 }}
68-
# dp-attn: ${{ matrix.config.dp-attn || false }}
69-
# conc: ${{ matrix.config.conc }}
51+
benchmark-70b:
52+
needs: get-70b-configs
53+
uses: ./.github/workflows/benchmark-tmpl.yml
54+
name: 70b 1k1k
55+
strategy:
56+
fail-fast: false
57+
matrix:
58+
config: ${{ fromJson(needs.get-70b-configs.outputs.search-space-config) }}
59+
secrets: inherit
60+
with:
61+
exp-name: "70b_1k1k"
62+
isl: 1024
63+
osl: 1024
64+
max-model-len: 2048
65+
runner: ${{ matrix.config.runner }}
66+
image: ${{ matrix.config.image }}
67+
model: ${{ matrix.config.model }}
68+
framework: ${{ matrix.config.framework }}
69+
precision: ${{ matrix.config.precision }}
70+
tp: ${{ matrix.config.tp }}
71+
ep: ${{ matrix.config.ep || 1 }}
72+
dp-attn: ${{ matrix.config.dp-attn || false }}
73+
conc: ${{ matrix.config.conc }}
7074

71-
# benchmark-dsr1:
72-
# needs: get-dsr1-configs
73-
# uses: ./.github/workflows/benchmark-tmpl.yml
74-
# name: dsr1 1k1k
75-
# strategy:
76-
# fail-fast: false
77-
# matrix:
78-
# config: ${{ fromJson(needs.get-dsr1-configs.outputs.search-space-config) }}
79-
# secrets: inherit
80-
# with:
81-
# exp-name: "dsr1_1k1k"
82-
# isl: 1024
83-
# osl: 1024
84-
# max-model-len: 2048
85-
# runner: ${{ matrix.config.runner }}
86-
# image: ${{ matrix.config.image }}
87-
# model: ${{ matrix.config.model }}
88-
# framework: ${{ matrix.config.framework }}
89-
# precision: ${{ matrix.config.precision }}
90-
# tp: ${{ matrix.config.tp }}
91-
# ep: ${{ matrix.config.ep || 1 }}
92-
# dp-attn: ${{ matrix.config.dp-attn || false }}
93-
# conc: ${{ matrix.config.conc }}
75+
benchmark-dsr1:
76+
needs: get-dsr1-configs
77+
uses: ./.github/workflows/benchmark-tmpl.yml
78+
name: dsr1 1k1k
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
config: ${{ fromJson(needs.get-dsr1-configs.outputs.search-space-config) }}
83+
secrets: inherit
84+
with:
85+
exp-name: "dsr1_1k1k"
86+
isl: 1024
87+
osl: 1024
88+
max-model-len: 2048
89+
runner: ${{ matrix.config.runner }}
90+
image: ${{ matrix.config.image }}
91+
model: ${{ matrix.config.model }}
92+
framework: ${{ matrix.config.framework }}
93+
precision: ${{ matrix.config.precision }}
94+
tp: ${{ matrix.config.tp }}
95+
ep: ${{ matrix.config.ep || 1 }}
96+
dp-attn: ${{ matrix.config.dp-attn || false }}
97+
conc: ${{ matrix.config.conc }}
9498

9599
benchmark-gptoss:
96100
needs: get-gptoss-configs
@@ -116,21 +120,21 @@ jobs:
116120
dp-attn: ${{ matrix.config.dp-attn || false }}
117121
conc: ${{ matrix.config.conc }}
118122

119-
# collect-70b-results:
120-
# needs: benchmark-70b
121-
# if: ${{ always() }}
122-
# uses: ./.github/workflows/collect-results.yml
123-
# secrets: inherit
124-
# with:
125-
# exp-name: "70b_1k1k"
123+
collect-70b-results:
124+
needs: benchmark-70b
125+
if: ${{ always() }}
126+
uses: ./.github/workflows/collect-results.yml
127+
secrets: inherit
128+
with:
129+
exp-name: "70b_1k1k"
126130

127-
# collect-dsr1-results:
128-
# needs: benchmark-dsr1
129-
# if: ${{ always() }}
130-
# uses: ./.github/workflows/collect-results.yml
131-
# secrets: inherit
132-
# with:
133-
# exp-name: "dsr1_1k1k"
131+
collect-dsr1-results:
132+
needs: benchmark-dsr1
133+
if: ${{ always() }}
134+
uses: ./.github/workflows/collect-results.yml
135+
secrets: inherit
136+
with:
137+
exp-name: "dsr1_1k1k"
134138

135139
collect-gptoss-results:
136140
needs: benchmark-gptoss

‎.github/workflows/1k8k-sweep.yml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: "1K/8K Sweep"
2+
3+
concurrency:
4+
group: benchmark-lock-1k8k
5+
cancel-in-progress: false
6+
7+
on:
8+
# pull_request:
9+
workflow_dispatch:
10+
11+
jobs:
12+
get-70b-configs:
13+
runs-on: ubuntu-latest
14+
outputs:
15+
search-space-config: ${{ steps.get-70b-configs.outputs.search-space-config }}
16+
steps:
17+
- name: Checkout code
18+
uses: actions/checkout@v4
19+
20+
- id: get-70b-configs
21+
run: |
22+
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/get_full_sweep_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix 70b)
23+
echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
24+
25+
get-dsr1-configs:
26+
runs-on: ubuntu-latest
27+
outputs:
28+
search-space-config: ${{ steps.get-dsr1-configs.outputs.search-space-config }}
29+
steps:
30+
- name: Checkout code
31+
uses: actions/checkout@v4
32+
33+
- id: get-dsr1-configs
34+
run: |
35+
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/get_full_sweep_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix dsr1)
36+
echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
37+
38+
get-gptoss-configs:
39+
runs-on: ubuntu-latest
40+
outputs:
41+
search-space-config: ${{ steps.get-gptoss-configs.outputs.search-space-config }}
42+
steps:
43+
- name: Checkout code
44+
uses: actions/checkout@v4
45+
46+
- id: get-gptoss-configs
47+
run: |
48+
CONFIG_JSON=$(python3 ${GITHUB_WORKSPACE}/utils/matrix-logic/get_full_sweep_configs.py --config-files ${GITHUB_WORKSPACE}/.github/configs/nvidia-master.yaml ${GITHUB_WORKSPACE}/.github/configs/amd-master.yaml --seq-lens 1k8k --model-prefix gptoss)
49+
echo "search-space-config=$CONFIG_JSON" >> $GITHUB_OUTPUT
50+
51+
benchmark-70b:
52+
needs: get-70b-configs
53+
uses: ./.github/workflows/benchmark-tmpl.yml
54+
name: 70b 1k8k
55+
strategy:
56+
fail-fast: false
57+
matrix:
58+
config: ${{ fromJson(needs.get-70b-configs.outputs.search-space-config) }}
59+
secrets: inherit
60+
with:
61+
exp-name: "70b_1k8k"
62+
isl: 1024
63+
osl: 1024
64+
max-model-len: 2048
65+
runner: ${{ matrix.config.runner }}
66+
image: ${{ matrix.config.image }}
67+
model: ${{ matrix.config.model }}
68+
framework: ${{ matrix.config.framework }}
69+
precision: ${{ matrix.config.precision }}
70+
tp: ${{ matrix.config.tp }}
71+
ep: ${{ matrix.config.ep || 1 }}
72+
dp-attn: ${{ matrix.config.dp-attn || false }}
73+
conc: ${{ matrix.config.conc }}
74+
75+
benchmark-dsr1:
76+
needs: get-dsr1-configs
77+
uses: ./.github/workflows/benchmark-tmpl.yml
78+
name: dsr1 1k8k
79+
strategy:
80+
fail-fast: false
81+
matrix:
82+
config: ${{ fromJson(needs.get-dsr1-configs.outputs.search-space-config) }}
83+
secrets: inherit
84+
with:
85+
exp-name: "dsr1_1k8k"
86+
isl: 1024
87+
osl: 1024
88+
max-model-len: 2048
89+
runner: ${{ matrix.config.runner }}
90+
image: ${{ matrix.config.image }}
91+
model: ${{ matrix.config.model }}
92+
framework: ${{ matrix.config.framework }}
93+
precision: ${{ matrix.config.precision }}
94+
tp: ${{ matrix.config.tp }}
95+
ep: ${{ matrix.config.ep || 1 }}
96+
dp-attn: ${{ matrix.config.dp-attn || false }}
97+
conc: ${{ matrix.config.conc }}
98+
99+
benchmark-gptoss:
100+
needs: get-gptoss-configs
101+
uses: ./.github/workflows/benchmark-tmpl.yml
102+
name: gptoss 1k8k
103+
strategy:
104+
fail-fast: false
105+
matrix:
106+
config: ${{ fromJson(needs.get-gptoss-configs.outputs.search-space-config) }}
107+
secrets: inherit
108+
with:
109+
exp-name: "gptoss_1k8k"
110+
isl: 1024
111+
osl: 1024
112+
max-model-len: 2048
113+
runner: ${{ matrix.config.runner }}
114+
image: ${{ matrix.config.image }}
115+
model: ${{ matrix.config.model }}
116+
framework: ${{ matrix.config.framework }}
117+
precision: ${{ matrix.config.precision }}
118+
tp: ${{ matrix.config.tp }}
119+
ep: ${{ matrix.config.ep || 1 }}
120+
dp-attn: ${{ matrix.config.dp-attn || false }}
121+
conc: ${{ matrix.config.conc }}
122+
123+
collect-70b-results:
124+
needs: benchmark-70b
125+
if: ${{ always() }}
126+
uses: ./.github/workflows/collect-results.yml
127+
secrets: inherit
128+
with:
129+
exp-name: "70b_1k8k"
130+
131+
collect-dsr1-results:
132+
needs: benchmark-dsr1
133+
if: ${{ always() }}
134+
uses: ./.github/workflows/collect-results.yml
135+
secrets: inherit
136+
with:
137+
exp-name: "dsr1_1k8k"
138+
139+
collect-gptoss-results:
140+
needs: benchmark-gptoss
141+
if: ${{ always() }}
142+
uses: ./.github/workflows/collect-results.yml
143+
secrets: inherit
144+
with:
145+
exp-name: "gptoss_1k8k"

0 commit comments

Comments
 (0)