Skip to content

Commit 437b56b

Browse files
hshrivastava-droidhshrivastava-droid
authored
Merge branch 'main' into dsv4-fp4-gb300-1k1k-stp-disagg
2 parents 7f63c31 + d527fc8 commit 437b56b

51 files changed

Lines changed: 6887 additions & 296 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/benchmark-tmpl.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ jobs:
145145
timeout-minutes: 500
146146
name: >-
147147
${{ inputs.model-prefix }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework == 'sglang' && 'sgl' || inputs.framework == 'dynamo-sglang' && 'dyn-sgl' || inputs.framework == 'sglang-disagg' && 'sgl-disagg' || inputs.framework }}
148-
TP${{ inputs.tp }}/DCP${{ inputs.dcp-size }}/PCP${{ inputs.pcp-size }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }}
148+
TP${{ inputs.tp }}${{ inputs.dcp-size != '' && inputs.dcp-size != '1' && format('/DCP{0}', inputs.dcp-size) || '' }}${{ inputs.pcp-size != '' && inputs.pcp-size != '1' && format('/PCP{0}', inputs.pcp-size) || '' }}${{ inputs.ep != '' && inputs.ep != '1' && format('/EP{0}', inputs.ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }}
149149
${{ inputs.spec-decoding != 'none' && inputs.spec-decoding || '' }}
150150
${{ inputs.kv-offloading != '' && inputs.kv-offloading != 'none' && format('{0} KV offload', inputs.kv-offloading) || '' }}
151151
${{ inputs.kv-offload-backend != '' && inputs.kv-offload-backend != 'none' && inputs.kv-offload-backend != 'default' && inputs.kv-offload-backend || '' }}

‎.github/workflows/profile.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ jobs:
100100
config: ${{ fromJson(needs.get-jobs.outputs.filtered-matrix) }}
101101
name: >-
102102
${{ matrix.config.model-prefix }} ${{ matrix.config.precision }} ${{ matrix.config.runner }} ${{ matrix.config.framework }}
103-
TP${{ matrix.config.tp }}/DCP${{ matrix.config.dcp-size }}/PCP${{ matrix.config.pcp-size }} c${{ matrix.config.conc }}
103+
TP${{ matrix.config.tp }}${{ matrix.config.dcp-size != '' && matrix.config.dcp-size != '1' && format('/DCP{0}', matrix.config.dcp-size) || '' }}${{ matrix.config.pcp-size != '' && matrix.config.pcp-size != '1' && format('/PCP{0}', matrix.config.pcp-size) || '' }} c${{ matrix.config.conc }}
104104
runs-on: ${{ matrix.config.runner }}
105105
env:
106106
EXP_NAME: ${{ matrix.config.exp-name }}

‎benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_lowlat[0]): 1p5d low-latency (dep4 prefill / tep8 decode, 5 decode nodes).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (sglang/<model>/<hw>-<precision>/<seq>/disagg/<variant>/...).
5+
6+
name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d
7+
model:
8+
path: dsr1
9+
container: dynamo-sglang
10+
precision: fp4
11+
12+
dynamo:
13+
hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
14+
install: true
15+
16+
resources:
17+
gpu_type: b200
18+
prefill_nodes: 1
19+
prefill_workers: 1
20+
gpus_per_prefill: 4
21+
decode_nodes: 5
22+
decode_workers: 5
23+
gpus_per_node: 8
24+
backend:
25+
prefill_environment:
26+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
27+
PYTHONUNBUFFERED: '1'
28+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
29+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
30+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
31+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
32+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
33+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
34+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
35+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
36+
MC_FORCE_MNNVL: '1'
37+
NCCL_MNNVL_ENABLE: '1'
38+
NCCL_CUMEM_ENABLE: '1'
39+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
40+
DYN_REQUEST_PLANE: nats
41+
SGLANG_ENABLE_SPEC_V2: '1'
42+
decode_environment:
43+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
44+
PYTHONUNBUFFERED: '1'
45+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
46+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
47+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
50+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
51+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
52+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
53+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
54+
MC_FORCE_MNNVL: '1'
55+
NCCL_MNNVL_ENABLE: '1'
56+
NCCL_CUMEM_ENABLE: '1'
57+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
58+
DYN_REQUEST_PLANE: nats
59+
SGLANG_ENABLE_SPEC_V2: '1'
60+
sglang_config:
61+
prefill:
62+
served-model-name: deepseek-ai/DeepSeek-R1
63+
trust-remote-code: true
64+
quantization: modelopt_fp4
65+
disaggregation-mode: prefill
66+
disaggregation-transfer-backend: nixl
67+
mem-fraction-static: 0.85
68+
max-prefill-tokens: 32768
69+
chunked-prefill-size: 32768
70+
context-length: 2200
71+
max-running-requests: 512
72+
disable-cuda-graph: true
73+
tensor-parallel-size: 4
74+
data-parallel-size: 4
75+
expert-parallel-size: 4
76+
enable-dp-attention: true
77+
enable-dp-lm-head: true
78+
attention-backend: trtllm_mla
79+
kv-cache-dtype: fp8_e4m3
80+
moe-runner-backend: flashinfer_trtllm
81+
moe-dense-tp-size: 1
82+
stream-interval: 30
83+
watchdog-timeout: 1000000
84+
enable-flashinfer-allreduce-fusion: true
85+
disable-radix-cache: true
86+
fp4-gemm-backend: flashinfer_trtllm
87+
decode:
88+
served-model-name: deepseek-ai/DeepSeek-R1
89+
trust-remote-code: true
90+
quantization: modelopt_fp4
91+
disaggregation-mode: decode
92+
disaggregation-transfer-backend: nixl
93+
mem-fraction-static: 0.85
94+
max-prefill-tokens: 32768
95+
chunked-prefill-size: 32768
96+
context-length: 2200
97+
max-running-requests: 512
98+
cuda-graph-max-bs: 512
99+
tensor-parallel-size: 8
100+
data-parallel-size: 1
101+
expert-parallel-size: 8
102+
attention-backend: trtllm_mla
103+
kv-cache-dtype: fp8_e4m3
104+
moe-runner-backend: flashinfer_trtllm
105+
stream-interval: 30
106+
watchdog-timeout: 1000000
107+
enable-flashinfer-allreduce-fusion: true
108+
disable-radix-cache: true
109+
fp4-gemm-backend: flashinfer_trtllm
110+
speculative-algorithm: EAGLE
111+
speculative-num-steps: 2
112+
speculative-eagle-topk: 1
113+
speculative-num-draft-tokens: 3
114+
health_check:
115+
max_attempts: 360
116+
interval_seconds: 10
117+
benchmark:
118+
type: sa-bench
119+
isl: 1024
120+
osl: 1024
121+
req_rate: inf
122+
concurrencies: 16x512
123+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_lowlat[1]): 1p6d low-latency (dep4 prefill / tep8 decode, 6 decode nodes).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (sglang/<model>/<hw>-<precision>/<seq>/disagg/<variant>/...).
5+
6+
name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d
7+
model:
8+
path: dsr1
9+
container: dynamo-sglang
10+
precision: fp4
11+
12+
dynamo:
13+
hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
14+
install: true
15+
16+
resources:
17+
gpu_type: b200
18+
prefill_nodes: 1
19+
prefill_workers: 1
20+
gpus_per_prefill: 4
21+
decode_nodes: 6
22+
decode_workers: 6
23+
gpus_per_node: 8
24+
backend:
25+
prefill_environment:
26+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
27+
PYTHONUNBUFFERED: '1'
28+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
29+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
30+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
31+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
32+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
33+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
34+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
35+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
36+
MC_FORCE_MNNVL: '1'
37+
NCCL_MNNVL_ENABLE: '1'
38+
NCCL_CUMEM_ENABLE: '1'
39+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
40+
DYN_REQUEST_PLANE: nats
41+
SGLANG_ENABLE_SPEC_V2: '1'
42+
decode_environment:
43+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
44+
PYTHONUNBUFFERED: '1'
45+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
46+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
47+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
50+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
51+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
52+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
53+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
54+
MC_FORCE_MNNVL: '1'
55+
NCCL_MNNVL_ENABLE: '1'
56+
NCCL_CUMEM_ENABLE: '1'
57+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
58+
DYN_REQUEST_PLANE: nats
59+
SGLANG_ENABLE_SPEC_V2: '1'
60+
sglang_config:
61+
prefill:
62+
served-model-name: deepseek-ai/DeepSeek-R1
63+
trust-remote-code: true
64+
quantization: modelopt_fp4
65+
disaggregation-mode: prefill
66+
disaggregation-transfer-backend: nixl
67+
mem-fraction-static: 0.85
68+
max-prefill-tokens: 32768
69+
chunked-prefill-size: 32768
70+
context-length: 2200
71+
max-running-requests: 512
72+
disable-cuda-graph: true
73+
tensor-parallel-size: 4
74+
data-parallel-size: 4
75+
expert-parallel-size: 4
76+
enable-dp-attention: true
77+
enable-dp-lm-head: true
78+
attention-backend: trtllm_mla
79+
kv-cache-dtype: fp8_e4m3
80+
moe-runner-backend: flashinfer_trtllm
81+
moe-dense-tp-size: 1
82+
stream-interval: 30
83+
watchdog-timeout: 1000000
84+
enable-flashinfer-allreduce-fusion: true
85+
disable-radix-cache: true
86+
fp4-gemm-backend: flashinfer_trtllm
87+
decode:
88+
served-model-name: deepseek-ai/DeepSeek-R1
89+
trust-remote-code: true
90+
quantization: modelopt_fp4
91+
disaggregation-mode: decode
92+
disaggregation-transfer-backend: nixl
93+
mem-fraction-static: 0.85
94+
max-prefill-tokens: 32768
95+
chunked-prefill-size: 32768
96+
context-length: 2200
97+
max-running-requests: 512
98+
cuda-graph-max-bs: 512
99+
tensor-parallel-size: 8
100+
data-parallel-size: 1
101+
expert-parallel-size: 8
102+
attention-backend: trtllm_mla
103+
kv-cache-dtype: fp8_e4m3
104+
moe-runner-backend: flashinfer_trtllm
105+
stream-interval: 30
106+
watchdog-timeout: 1000000
107+
enable-flashinfer-allreduce-fusion: true
108+
disable-radix-cache: true
109+
fp4-gemm-backend: flashinfer_trtllm
110+
speculative-algorithm: EAGLE
111+
speculative-num-steps: 2
112+
speculative-eagle-topk: 1
113+
speculative-num-draft-tokens: 3
114+
health_check:
115+
max_attempts: 360
116+
interval_seconds: 10
117+
benchmark:
118+
type: sa-bench
119+
isl: 1024
120+
osl: 1024
121+
req_rate: inf
122+
concurrencies: 32x64x256x512
123+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml
2+
# base + zip_override_mtp_maxtpt[0]): 1p1d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.75).
3+
# One flat YAML per concrete topology, matching the 8k1k local recipe layout
4+
# (sglang/<model>/<hw>-<precision>/<seq>/disagg/<variant>/...).
5+
6+
name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d
7+
model:
8+
path: dsr1
9+
container: dynamo-sglang
10+
precision: fp4
11+
12+
dynamo:
13+
hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727"
14+
install: true
15+
16+
resources:
17+
gpu_type: b200
18+
prefill_nodes: 1
19+
prefill_workers: 1
20+
gpus_per_prefill: 4
21+
decode_nodes: 1
22+
decode_workers: 1
23+
gpus_per_node: 8
24+
backend:
25+
prefill_environment:
26+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
27+
PYTHONUNBUFFERED: '1'
28+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
29+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
30+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
31+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
32+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
33+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
34+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
35+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
36+
MC_FORCE_MNNVL: '1'
37+
NCCL_MNNVL_ENABLE: '1'
38+
NCCL_CUMEM_ENABLE: '1'
39+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
40+
DYN_REQUEST_PLANE: nats
41+
SGLANG_ENABLE_SPEC_V2: '1'
42+
decode_environment:
43+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800'
44+
PYTHONUNBUFFERED: '1'
45+
DYN_SKIP_SGLANG_LOG_FORMATTING: '1'
46+
SGLANG_ENABLE_JIT_DEEPGEMM: 'false'
47+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000'
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000'
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000'
50+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000'
51+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True'
52+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0'
53+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1'
54+
MC_FORCE_MNNVL: '1'
55+
NCCL_MNNVL_ENABLE: '1'
56+
NCCL_CUMEM_ENABLE: '1'
57+
SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1'
58+
DYN_REQUEST_PLANE: nats
59+
SGLANG_MOE_NVFP4_DISPATCH: '1'
60+
SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass
61+
SGLANG_ENABLE_SPEC_V2: '1'
62+
sglang_config:
63+
prefill:
64+
served-model-name: deepseek-ai/DeepSeek-R1
65+
trust-remote-code: true
66+
quantization: modelopt_fp4
67+
disaggregation-mode: prefill
68+
disaggregation-transfer-backend: nixl
69+
mem-fraction-static: 0.85
70+
max-prefill-tokens: 32768
71+
chunked-prefill-size: 32768
72+
context-length: 2200
73+
max-running-requests: 1024
74+
disable-cuda-graph: true
75+
tensor-parallel-size: 4
76+
data-parallel-size: 4
77+
expert-parallel-size: 4
78+
enable-dp-attention: true
79+
enable-dp-lm-head: true
80+
attention-backend: trtllm_mla
81+
kv-cache-dtype: fp8_e4m3
82+
moe-runner-backend: flashinfer_trtllm
83+
moe-dense-tp-size: 1
84+
stream-interval: 30
85+
watchdog-timeout: 1000000
86+
enable-flashinfer-allreduce-fusion: true
87+
disable-radix-cache: true
88+
fp4-gemm-backend: flashinfer_trtllm
89+
decode:
90+
served-model-name: deepseek-ai/DeepSeek-R1
91+
trust-remote-code: true
92+
quantization: modelopt_fp4
93+
disaggregation-mode: decode
94+
disaggregation-transfer-backend: nixl
95+
mem-fraction-static: 0.75
96+
max-prefill-tokens: 32768
97+
chunked-prefill-size: 32768
98+
context-length: 2200
99+
max-running-requests: 1024
100+
cuda-graph-max-bs: 1024
101+
tensor-parallel-size: 8
102+
data-parallel-size: 8
103+
expert-parallel-size: 8
104+
attention-backend: trtllm_mla
105+
kv-cache-dtype: fp8_e4m3
106+
moe-runner-backend: flashinfer_trtllm
107+
stream-interval: 30
108+
watchdog-timeout: 1000000
109+
enable-flashinfer-allreduce-fusion: true
110+
disable-radix-cache: true
111+
fp4-gemm-backend: flashinfer_trtllm
112+
enable-dp-attention: true
113+
enable-dp-lm-head: true
114+
moe-dense-tp-size: 1
115+
speculative-algorithm: EAGLE
116+
speculative-num-steps: 2
117+
speculative-eagle-topk: 1
118+
speculative-num-draft-tokens: 3
119+
health_check:
120+
max_attempts: 360
121+
interval_seconds: 10
122+
benchmark:
123+
type: sa-bench
124+
isl: 1024
125+
osl: 1024
126+
req_rate: inf
127+
concurrencies: 512x1024
128+
use_chat_template: true

0 commit comments

Comments
 (0)