Skip to content

Commit 29f2f8c

Browse files
RohitNagrajRohitNagraj
committed
Add DSV4 FP4 B200 Dynamo SGLang config
1 parent 1438043 commit 29f2f8c

13 files changed

Lines changed: 1703 additions & 16 deletions

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-dep8.yaml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: "disagg-b200-1p1d-dep8-dep8-c512"
2+
3+
4+
model:
5+
path: "deepseek-v4-pro"
6+
container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494"
7+
precision: "fp4"
8+
9+
dynamo:
10+
hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e"
11+
install: true
12+
13+
sbatch_directives:
14+
cpus-per-task: "144"
15+
mem: "0"
16+
17+
resources:
18+
gpu_type: "b200"
19+
gpus_per_node: 8
20+
prefill_nodes: 1
21+
prefill_workers: 1
22+
gpus_per_prefill: 8
23+
decode_nodes: 1
24+
decode_workers: 1
25+
gpus_per_decode: 8
26+
27+
health_check:
28+
max_attempts: 240
29+
frontend:
30+
type: dynamo
31+
enable_multiple_frontends: false
32+
env:
33+
DYN_ROUTER_LOAD_BLOCK_SIZE: "1"
34+
args:
35+
router-mode: "kv"
36+
router-kv-overlap-score-weight: 0
37+
router-queue-threshold: 64
38+
router-temperature: 0.5
39+
no-kv-events: true
40+
41+
backend:
42+
type: sglang
43+
44+
prefill_environment:
45+
PYTHONUNBUFFERED: "1"
46+
SGLANG_RADIX_FORCE_MISS: "1"
47+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
48+
SGLANG_ENABLE_THINKING: "1"
49+
SGLANG_REASONING_EFFORT: "max"
50+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
51+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
52+
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
53+
SGLANG_OPT_USE_FAST_MASK_EP: "1"
54+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
55+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
56+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
57+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
58+
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
59+
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
60+
SGLANG_OPT_FP8_WO_A_GEMM: "1"
61+
NCCL_MNNVL_ENABLE: "1"
62+
NCCL_CUMEM_ENABLE: "1"
63+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
64+
MC_FORCE_MNNVL: "1"
65+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
66+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
67+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
68+
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
69+
SGLANG_LOG_FORWARD_ITERS: "1"
70+
SGLANG_LOG_MS: "1"
71+
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
72+
73+
decode_environment:
74+
PYTHONUNBUFFERED: "1"
75+
SGLANG_RADIX_FORCE_MISS: "1"
76+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
77+
SGLANG_ENABLE_THINKING: "1"
78+
SGLANG_REASONING_EFFORT: "max"
79+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
80+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
81+
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
82+
SGLANG_OPT_USE_FAST_MASK_EP: "1"
83+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
84+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
85+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
86+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
87+
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
88+
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
89+
NCCL_MNNVL_ENABLE: "1"
90+
NCCL_CUMEM_ENABLE: "1"
91+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
92+
SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8"
93+
MC_FORCE_MNNVL: "1"
94+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
95+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
96+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
97+
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
98+
SGLANG_LOG_FORWARD_ITERS: "1"
99+
SGLANG_LOG_MS: "1"
100+
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
101+
102+
sglang_config:
103+
prefill:
104+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
105+
trust-remote-code: true
106+
watchdog-timeout: 86400
107+
skip-tokenizer-init: true
108+
stream-interval: 60
109+
110+
tensor-parallel-size: 8
111+
data-parallel-size: 8
112+
expert-parallel-size: 8
113+
114+
enable-dp-attention: true
115+
moe-a2a-backend: "megamoe"
116+
deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}'
117+
moe-dense-tp-size: 1
118+
119+
disaggregation-mode: "prefill"
120+
disaggregation-transfer-backend: mooncake
121+
122+
mem-fraction-static: 0.80
123+
max-running-requests: 1024
124+
chunked-prefill-size: 65536
125+
126+
decode:
127+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
128+
trust-remote-code: true
129+
watchdog-timeout: 86400
130+
skip-tokenizer-init: true
131+
stream-interval: 60
132+
133+
load-balance-method: "total_requests"
134+
moe-a2a-backend: "megamoe"
135+
136+
disaggregation-mode: "decode"
137+
disaggregation-transfer-backend: mooncake
138+
disaggregation-decode-polling-interval: 8
139+
140+
mem-fraction-static: 0.94
141+
swa-full-tokens-ratio: 0.056
142+
context-length: 9216
143+
tensor-parallel-size: 8
144+
data-parallel-size: 8
145+
expert-parallel-size: 8
146+
enable-dp-attention: true
147+
enable-dp-lm-head: true
148+
max-running-requests: 21504
149+
cuda-graph-max-bs: 1280
150+
151+
152+
benchmark:
153+
type: "sa-bench"
154+
isl: 8192
155+
osl: 1024
156+
concurrencies: "256x512"
157+
req_rate: "inf"
158+
use_chat_template: false

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b200-1p1d-dep8-tp8.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: "disagg-b200-1p1d-dep8-tp8-4-c64"
2+
3+
4+
model:
5+
path: "deepseek-v4-pro"
6+
container: "lmsysorg/sglang:nightly-dev-cu13-20260708-b3632494"
7+
precision: "fp4"
8+
9+
dynamo:
10+
hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e"
11+
install: true
12+
13+
sbatch_directives:
14+
cpus-per-task: "144"
15+
mem: "0"
16+
17+
resources:
18+
gpu_type: "b200"
19+
gpus_per_node: 8
20+
prefill_nodes: 1
21+
prefill_workers: 1
22+
gpus_per_prefill: 8
23+
decode_nodes: 1
24+
decode_workers: 1
25+
gpus_per_decode: 8
26+
27+
health_check:
28+
max_attempts: 240
29+
30+
frontend:
31+
type: dynamo
32+
enable_multiple_frontends: false
33+
env:
34+
DYN_ROUTER_LOAD_BLOCK_SIZE: "1"
35+
args:
36+
router-mode: "kv"
37+
router-kv-overlap-score-weight: 0
38+
router-queue-threshold: 64
39+
router-temperature: 0.5
40+
no-kv-events: true
41+
42+
backend:
43+
type: sglang
44+
45+
prefill_environment:
46+
PYTHONUNBUFFERED: "1"
47+
SGLANG_RADIX_FORCE_MISS: "1"
48+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
49+
SGLANG_ENABLE_THINKING: "1"
50+
SGLANG_REASONING_EFFORT: "max"
51+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
52+
SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1"
53+
SGLANG_OPT_FIX_HASH_MEGA_MOE: "1"
54+
SGLANG_OPT_USE_FAST_MASK_EP: "1"
55+
SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1"
56+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
57+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
58+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
59+
SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1"
60+
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
61+
SGLANG_OPT_FP8_WO_A_GEMM: "1"
62+
NCCL_MNNVL_ENABLE: "1"
63+
NCCL_CUMEM_ENABLE: "1"
64+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
65+
MC_FORCE_MNNVL: "1"
66+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
67+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
68+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
69+
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
70+
SGLANG_LOG_FORWARD_ITERS: "1"
71+
SGLANG_LOG_MS: "1"
72+
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
73+
74+
decode_environment:
75+
PYTHONUNBUFFERED: "1"
76+
SGLANG_RADIX_FORCE_MISS: "1"
77+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
78+
SGLANG_ENABLE_THINKING: "1"
79+
SGLANG_REASONING_EFFORT: "max"
80+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
81+
SGLANG_OPT_USE_JIT_NORM: "1"
82+
SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1"
83+
SGLANG_OPT_USE_TOPK_V2: "1"
84+
NCCL_MNNVL_ENABLE: "1"
85+
NCCL_CUMEM_ENABLE: "1"
86+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
87+
MC_FORCE_MNNVL: "1"
88+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
89+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
90+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
91+
92+
sglang_config:
93+
prefill:
94+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
95+
trust-remote-code: true
96+
watchdog-timeout: 86400
97+
skip-tokenizer-init: true
98+
stream-interval: 60
99+
100+
tensor-parallel-size: 8
101+
data-parallel-size: 8
102+
expert-parallel-size: 8
103+
104+
enable-dp-attention: true
105+
moe-a2a-backend: "megamoe"
106+
deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}'
107+
moe-dense-tp-size: 1
108+
109+
disaggregation-mode: "prefill"
110+
disaggregation-transfer-backend: mooncake
111+
112+
mem-fraction-static: 0.80
113+
max-running-requests: 1024
114+
chunked-prefill-size: 65536
115+
116+
decode:
117+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
118+
model-path: "/model/"
119+
trust-remote-code: true
120+
disable-radix-cache: true
121+
122+
disaggregation-mode: "decode"
123+
disaggregation-transfer-backend: mooncake
124+
125+
tensor-parallel-size: 8
126+
data-parallel-size: 1
127+
expert-parallel-size: 1
128+
129+
moe-runner-backend: "flashinfer_mxfp4"
130+
disable-flashinfer-autotune: true
131+
132+
mem-fraction-static: 0.9
133+
max-running-requests: 1024
134+
cuda-graph-max-bs: 512
135+
swa-full-tokens-ratio: 0.1
136+
context-length: 16384
137+
138+
139+
benchmark:
140+
type: "sa-bench"
141+
isl: 8192
142+
osl: 1024
143+
concurrencies: "32x64"
144+
req_rate: "inf"
145+
use_chat_template: false

0 commit comments

Comments
 (0)