Skip to content

Commit 3ca4ba9

Browse files
RohitNagrajRohitNagraj
committed
feat: add DSV4 FP4 B200 Dynamo SGLang MTP config
Add five recipe-backed 8k/1k MTP topologies and config-scoped B200 launcher handling. 中文：新增 DSV4 FP4 B200 分离式 Dynamo SGLang MTP 配置，加入五个由 recipe 驱动的 8k/1k MTP 拓扑，并将 B200 启动器处理限定到该配置。
1 parent 1438043 commit 3ca4ba9

8 files changed

Lines changed: 789 additions & 14 deletions

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p1d-tp8-tp8-mtp.yaml‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
name: "disagg-b200-8k1k-low-latency-1p1d-tp8-tp8-mtp"
2+
3+
frontend:
4+
type: dynamo
5+
enable_multiple_frontends: true
6+
num_additional_frontends: 8
7+
8+
dynamo:
9+
hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e"
10+
install: true
11+
12+
model:
13+
path: "deepseek-v4-pro"
14+
container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494"
15+
precision: "fp4"
16+
17+
sbatch_directives:
18+
cpus-per-task: "144"
19+
mem: "0"
20+
21+
resources:
22+
gpu_type: "b200"
23+
gpus_per_node: 8
24+
prefill_nodes: 1
25+
prefill_workers: 1
26+
gpus_per_prefill: 8
27+
decode_nodes: 1
28+
decode_workers: 1
29+
gpus_per_decode: 8
30+
31+
health_check:
32+
max_attempts: 240
33+
34+
backend:
35+
type: sglang
36+
37+
prefill_environment:
38+
PYTHONUNBUFFERED: "1"
39+
SGLANG_RADIX_FORCE_MISS: "1"
40+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
41+
SGLANG_DEFAULT_THINKING: "1"
42+
SGLANG_DSV4_REASONING_EFFORT: "max"
43+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
44+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
45+
NCCL_MNNVL_ENABLE: "1"
46+
NCCL_CUMEM_ENABLE: "1"
47+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
48+
MC_FORCE_MNNVL: "1"
49+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
50+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
51+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
52+
53+
decode_environment:
54+
PYTHONUNBUFFERED: "1"
55+
SGLANG_RADIX_FORCE_MISS: "1"
56+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
57+
SGLANG_DEFAULT_THINKING: "1"
58+
SGLANG_DSV4_REASONING_EFFORT: "max"
59+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
60+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
61+
NCCL_MNNVL_ENABLE: "1"
62+
NCCL_CUMEM_ENABLE: "1"
63+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
64+
MC_FORCE_MNNVL: "1"
65+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
66+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
67+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
68+
69+
sglang_config:
70+
prefill:
71+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
72+
model-path: "/model/"
73+
trust-remote-code: true
74+
tool-call-parser: deepseekv4
75+
76+
disaggregation-mode: "prefill"
77+
disaggregation-transfer-backend: mooncake
78+
79+
tensor-parallel-size: 8
80+
data-parallel-size: 1
81+
expert-parallel-size: 1
82+
83+
moe-runner-backend: "flashinfer_mxfp4"
84+
disable-flashinfer-autotune: true
85+
86+
mem-fraction-static: 0.9
87+
max-running-requests: 16
88+
cuda-graph-max-bs: 8
89+
chunked-prefill-size: 65536
90+
91+
decode:
92+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
93+
model-path: "/model/"
94+
trust-remote-code: true
95+
tool-call-parser: deepseekv4
96+
97+
disaggregation-mode: "decode"
98+
disaggregation-transfer-backend: mooncake
99+
100+
tensor-parallel-size: 8
101+
data-parallel-size: 1
102+
expert-parallel-size: 1
103+
104+
moe-runner-backend: "flashinfer_mxfp4"
105+
disable-flashinfer-autotune: true
106+
107+
speculative-algo: "EAGLE"
108+
speculative-num-steps: 3
109+
speculative-eagle-topk: 1
110+
speculative-num-draft-tokens: 4
111+
112+
mem-fraction-static: 0.9
113+
max-running-requests: 8
114+
cuda-graph-max-bs: 8
115+
swa-full-tokens-ratio: 0.1
116+
context-length: 16384
117+
118+
benchmark:
119+
type: "sa-bench"
120+
isl: 8192
121+
osl: 1024
122+
random_range_ratio: 0.8
123+
concurrencies: "1"
124+
req_rate: "inf"
125+
use_chat_template: true
126+
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-low-latency-1p6d-dep8-tp8-mtp.yaml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
name: "disagg-b200-8k1k-low-latency-1p6d-dep8-tp8-mtp"
2+
3+
frontend:
4+
type: dynamo
5+
enable_multiple_frontends: true
6+
num_additional_frontends: 8
7+
8+
dynamo:
9+
hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e"
10+
install: true
11+
12+
model:
13+
path: "deepseek-v4-pro"
14+
container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494"
15+
precision: "fp4"
16+
17+
sbatch_directives:
18+
cpus-per-task: "144"
19+
mem: "0"
20+
21+
resources:
22+
gpu_type: "b200"
23+
gpus_per_node: 8
24+
prefill_nodes: 1
25+
prefill_workers: 1
26+
gpus_per_prefill: 8
27+
decode_nodes: 6
28+
decode_workers: 6
29+
gpus_per_decode: 8
30+
31+
health_check:
32+
max_attempts: 240
33+
34+
backend:
35+
type: sglang
36+
37+
prefill_environment:
38+
PYTHONUNBUFFERED: "1"
39+
SGLANG_RADIX_FORCE_MISS: "1"
40+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
41+
SGLANG_DEFAULT_THINKING: "1"
42+
SGLANG_DSV4_REASONING_EFFORT: "max"
43+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
44+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
45+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
46+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
47+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
48+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
49+
50+
NCCL_MNNVL_ENABLE: "1"
51+
NCCL_CUMEM_ENABLE: "1"
52+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
53+
MC_FORCE_MNNVL: "1"
54+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
55+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
56+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
57+
58+
decode_environment:
59+
PYTHONUNBUFFERED: "1"
60+
SGLANG_RADIX_FORCE_MISS: "1"
61+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
62+
SGLANG_DEFAULT_THINKING: "1"
63+
SGLANG_DSV4_REASONING_EFFORT: "max"
64+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
65+
NCCL_MNNVL_ENABLE: "1"
66+
NCCL_CUMEM_ENABLE: "1"
67+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
68+
MC_FORCE_MNNVL: "1"
69+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
70+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
71+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
72+
73+
sglang_config:
74+
prefill:
75+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
76+
model-path: "/model/"
77+
trust-remote-code: true
78+
tool-call-parser: deepseekv4
79+
80+
disaggregation-mode: "prefill"
81+
disaggregation-transfer-backend: mooncake
82+
83+
tensor-parallel-size: 8
84+
data-parallel-size: 8
85+
expert-parallel-size: 8
86+
87+
enable-dp-attention: true
88+
enable-dp-lm-head: true
89+
90+
moe-a2a-backend: "megamoe"
91+
deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
92+
93+
mem-fraction-static: 0.9
94+
max-running-requests: 256
95+
cuda-graph-max-bs: 128
96+
chunked-prefill-size: 65536
97+
98+
decode:
99+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
100+
model-path: "/model/"
101+
trust-remote-code: true
102+
tool-call-parser: deepseekv4
103+
104+
disaggregation-mode: "decode"
105+
disaggregation-transfer-backend: mooncake
106+
107+
tensor-parallel-size: 8
108+
data-parallel-size: 1
109+
expert-parallel-size: 1
110+
111+
moe-runner-backend: "flashinfer_mxfp4"
112+
disable-flashinfer-autotune: true
113+
114+
speculative-algo: "EAGLE"
115+
speculative-num-steps: 3
116+
speculative-eagle-topk: 1
117+
speculative-num-draft-tokens: 4
118+
119+
mem-fraction-static: 0.9
120+
max-running-requests: 128
121+
cuda-graph-max-bs: 128
122+
swa-full-tokens-ratio: 0.1
123+
context-length: 16384
124+
125+
benchmark:
126+
type: "sa-bench"
127+
isl: 8192
128+
osl: 1024
129+
random_range_ratio: 0.8
130+
concurrencies: "32x64x128"
131+
req_rate: "inf"
132+
use_chat_template: true
133+
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-mid-curve-1p1d-dep8-dep8-mtp.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
name: "disagg-b200-8k1k-mid-curve-1p1d-dep8-dep8-mtp"
2+
3+
frontend:
4+
type: dynamo
5+
enable_multiple_frontends: true
6+
num_additional_frontends: 8
7+
8+
dynamo:
9+
hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e"
10+
install: true
11+
12+
model:
13+
path: "deepseek-v4-pro"
14+
container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494"
15+
precision: "fp4"
16+
17+
sbatch_directives:
18+
cpus-per-task: "144"
19+
mem: "0"
20+
21+
resources:
22+
gpu_type: "b200"
23+
gpus_per_node: 8
24+
prefill_nodes: 1
25+
prefill_workers: 1
26+
gpus_per_prefill: 8
27+
decode_nodes: 1
28+
decode_workers: 1
29+
gpus_per_decode: 8
30+
31+
health_check:
32+
max_attempts: 240
33+
34+
backend:
35+
type: sglang
36+
37+
prefill_environment:
38+
PYTHONUNBUFFERED: "1"
39+
SGLANG_RADIX_FORCE_MISS: "1"
40+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
41+
SGLANG_DEFAULT_THINKING: "1"
42+
SGLANG_DSV4_REASONING_EFFORT: "max"
43+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
44+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
45+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
46+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
47+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
48+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
49+
NCCL_MNNVL_ENABLE: "1"
50+
NCCL_CUMEM_ENABLE: "1"
51+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
52+
MC_FORCE_MNNVL: "1"
53+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
54+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
55+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
56+
57+
decode_environment:
58+
PYTHONUNBUFFERED: "1"
59+
SGLANG_RADIX_FORCE_MISS: "1"
60+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
61+
SGLANG_DEFAULT_THINKING: "1"
62+
SGLANG_DSV4_REASONING_EFFORT: "max"
63+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
64+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
65+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096"
66+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
67+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
68+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
69+
NCCL_MNNVL_ENABLE: "1"
70+
NCCL_CUMEM_ENABLE: "1"
71+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
72+
MC_FORCE_MNNVL: "1"
73+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
74+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
75+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
76+
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0"
77+
78+
sglang_config:
79+
prefill:
80+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
81+
model-path: "/model/"
82+
trust-remote-code: true
83+
tool-call-parser: deepseekv4
84+
disaggregation-mode: "prefill"
85+
disaggregation-transfer-backend: mooncake
86+
tensor-parallel-size: 8
87+
data-parallel-size: 8
88+
expert-parallel-size: 8
89+
enable-dp-attention: true
90+
enable-dp-lm-head: true
91+
moe-a2a-backend: "megamoe"
92+
deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
93+
mem-fraction-static: 0.8
94+
max-running-requests: 1024
95+
cuda-graph-max-bs: 1024
96+
chunked-prefill-size: 65536
97+
stream-interval: 60
98+
99+
decode:
100+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
101+
model-path: "/model/"
102+
trust-remote-code: true
103+
tool-call-parser: deepseekv4
104+
disaggregation-mode: "decode"
105+
disaggregation-transfer-backend: mooncake
106+
tensor-parallel-size: 8
107+
data-parallel-size: 8
108+
expert-parallel-size: 8
109+
enable-dp-attention: true
110+
enable-dp-lm-head: true
111+
moe-a2a-backend: "megamoe"
112+
deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}'
113+
speculative-algo: "EAGLE"
114+
speculative-num-steps: 3
115+
speculative-eagle-topk: 1
116+
speculative-num-draft-tokens: 4
117+
mem-fraction-static: 0.9
118+
max-running-requests: 1536
119+
cuda-graph-max-bs: 1024
120+
swa-full-tokens-ratio: 0.15
121+
context-length: 16384
122+
stream-interval: 60
123+
124+
benchmark:
125+
type: "sa-bench"
126+
isl: 8192
127+
osl: 1024
128+
random_range_ratio: 0.8
129+
concurrencies: "256x1024"
130+
req_rate: "inf"
131+
use_chat_template: true
132+
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

0 commit comments

Comments
 (0)