Skip to content

Commit cd1cd4c

Browse files
RohitNagrajAnkur-singh
RohitNagraj
and
Ankur-singh
authored
Add Qwen3.5 FP8 GB300 Dynamo-SGLang MTP recipes / 新增 Qwen3.5 FP8 GB300 Dynamo-SGLang MTP 配方 (#2316)
* feat: add Qwen3.5 FP8 GB300 Dynamo-SGLang MTP recipes Add the 8k/1k multi-node configuration and seven recipes with Mooncake KV transfer and the pinned SGLang image. 中文：新增 Qwen3.5 FP8 GB300 Dynamo-SGLang MTP 的 8k/1k 多节点配置及七个配方，使用 Mooncake KV 传输和固定版本的 SGLang 镜像。 * chore: update perf changelog link for #2316 Record the public pull request URL in the appended changelog entry. 中文：在新增的性能变更日志条目中记录公开拉取请求链接。 * fix: enable chat templates for Qwen3.5 GB300 MTP Enable chat-formatted benchmark inputs in all seven Qwen3.5 FP8 GB300 MTP recipes and record the change in the existing changelog entry. 中文：为 Qwen3.5 GB300 MTP 启用聊天模板 在全部七个 Qwen3.5 FP8 GB300 MTP 配方中启用聊天格式的基准测试输入，并在现有变更日志条目中记录该变更。 * chore: refresh PR #2316 for sweep reuse [skip-sweep] --------- Co-authored-by: Ankur-singh <ankusingh@nvidia.com>
1 parent 648d857 commit cd1cd4c

9 files changed

Lines changed: 1296 additions & 0 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TP4/TP4 configuration.
2+
3+
name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tp4-tp4"
4+
5+
sbatch_directives:
6+
mem: "0"
7+
8+
dynamo:
9+
hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b
10+
install: true
11+
12+
frontend:
13+
type: dynamo
14+
enable_multiple_frontends: true
15+
num_additional_frontends: 1
16+
nginx_container: nginx
17+
18+
model:
19+
path: "qwen3.5-fp8"
20+
container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3"
21+
precision: "fp8"
22+
23+
resources:
24+
gpu_type: "gb300"
25+
gpus_per_node: 4
26+
prefill_nodes: 1
27+
decode_nodes: 1
28+
prefill_workers: 1
29+
decode_workers: 1
30+
31+
backend:
32+
type: sglang
33+
34+
prefill_environment:
35+
SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
36+
SGLANG_ENABLE_SPEC_V2: "1"
37+
NO_COLOR: "1"
38+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600"
39+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600"
40+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600"
41+
PYTHONUNBUFFERED: "1"
42+
NCCL_MNNVL_ENABLE: "1"
43+
NCCL_CUMEM_ENABLE: "1"
44+
MC_FORCE_MNNVL: "1"
45+
SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
46+
FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
47+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
50+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
51+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
52+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
53+
SGLANG_HEALTH_STARTING_OK: "1"
54+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0"
55+
56+
decode_environment:
57+
SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
58+
SGLANG_ENABLE_SPEC_V2: "1"
59+
NO_COLOR: "1"
60+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600"
61+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600"
62+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600"
63+
PYTHONUNBUFFERED: "1"
64+
NCCL_MNNVL_ENABLE: "1"
65+
NCCL_CUMEM_ENABLE: "1"
66+
MC_FORCE_MNNVL: "1"
67+
MC_TE_METRIC: "true"
68+
SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
69+
FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
70+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
71+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
72+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
73+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
74+
SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1"
75+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
76+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
77+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
78+
SGLANG_HEALTH_CHECK_TIMEOUT: "1800"
79+
SGLANG_HEALTH_STARTING_OK: "1"
80+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0"
81+
82+
sglang_config:
83+
prefill:
84+
served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
85+
model-path: "/model/"
86+
quantization: "fp8"
87+
kv-cache-dtype: "fp8_e4m3"
88+
trust-remote-code: true
89+
attention-backend: "trtllm_mha"
90+
tensor-parallel-size: 4
91+
mamba-scheduler-strategy: "no_buffer"
92+
mamba-track-interval: 2048
93+
mamba-ssm-dtype: "bfloat16"
94+
moe-runner-backend: "flashinfer_trtllm"
95+
disable-radix-cache: true
96+
max-running-requests: 1024
97+
mem-fraction-static: 0.8
98+
chunked-prefill-size: 16384
99+
max-prefill-tokens: 16384
100+
context-length: 16384
101+
cuda-graph-max-bs: 1024
102+
decode-log-interval: 1
103+
stream-interval: 50
104+
disaggregation-mode: "prefill"
105+
106+
decode:
107+
served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
108+
model-path: "/model/"
109+
quantization: "fp8"
110+
kv-cache-dtype: "fp8_e4m3"
111+
trust-remote-code: true
112+
attention-backend: "trtllm_mha"
113+
tensor-parallel-size: 4
114+
mamba-scheduler-strategy: "no_buffer"
115+
mamba-track-interval: 128
116+
mamba-ssm-dtype: "bfloat16"
117+
max-mamba-cache-size: 1024
118+
moe-runner-backend: "flashinfer_trtllm"
119+
120+
speculative-algorithm: "EAGLE"
121+
speculative-num-steps: 3
122+
speculative-eagle-topk: 1
123+
speculative-num-draft-tokens: 4
124+
125+
disable-radix-cache: true
126+
max-running-requests: 1024
127+
mem-fraction-static: 0.8
128+
chunked-prefill-size: 16384
129+
max-prefill-tokens: 16384
130+
context-length: 16384
131+
cuda-graph-max-bs: 1024
132+
decode-log-interval: 1
133+
stream-interval: 50
134+
disaggregation-mode: "decode"
135+
136+
benchmark:
137+
type: "sa-bench"
138+
isl: 8192
139+
osl: 1024
140+
req_rate: "inf"
141+
random_range_ratio: 0.8
142+
concurrencies: "1x2x8"
143+
use_chat_template: true

‎benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Qwen3.5 FP8 GB300 disaggregated MTP 1P1D TEP8/TEP8 configuration.
2+
3+
name: "qwen3.5-fp8-gb300-mtp-8k1k-1p1d-tep8-tep8"
4+
5+
sbatch_directives:
6+
mem: "0"
7+
8+
dynamo:
9+
hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b
10+
install: true
11+
12+
frontend:
13+
type: dynamo
14+
enable_multiple_frontends: true
15+
num_additional_frontends: 1
16+
nginx_container: nginx
17+
18+
model:
19+
path: "qwen3.5-fp8"
20+
container: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3"
21+
precision: "fp8"
22+
23+
resources:
24+
gpu_type: "gb300"
25+
gpus_per_node: 4
26+
prefill_nodes: 2
27+
decode_nodes: 2
28+
prefill_workers: 1
29+
decode_workers: 1
30+
31+
backend:
32+
type: sglang
33+
34+
prefill_environment:
35+
SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
36+
SGLANG_ENABLE_SPEC_V2: "1"
37+
NO_COLOR: "1"
38+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600"
39+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600"
40+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600"
41+
PYTHONUNBUFFERED: "1"
42+
NCCL_MNNVL_ENABLE: "1"
43+
NCCL_CUMEM_ENABLE: "1"
44+
MC_FORCE_MNNVL: "1"
45+
SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
46+
FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
47+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
50+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
51+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
52+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
53+
SGLANG_HEALTH_STARTING_OK: "1"
54+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0"
55+
56+
decode_environment:
57+
SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0"
58+
SGLANG_ENABLE_SPEC_V2: "1"
59+
NO_COLOR: "1"
60+
TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600"
61+
TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600"
62+
TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600"
63+
PYTHONUNBUFFERED: "1"
64+
NCCL_MNNVL_ENABLE: "1"
65+
NCCL_CUMEM_ENABLE: "1"
66+
MC_FORCE_MNNVL: "1"
67+
MC_TE_METRIC: "true"
68+
SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache"
69+
FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache"
70+
SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000"
71+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
72+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
73+
SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000"
74+
SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1"
75+
SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True"
76+
SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0"
77+
SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1"
78+
SGLANG_HEALTH_CHECK_TIMEOUT: "1800"
79+
SGLANG_HEALTH_STARTING_OK: "1"
80+
SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0"
81+
82+
sglang_config:
83+
prefill:
84+
served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
85+
model-path: "/model/"
86+
random-seed: 42
87+
quantization: "fp8"
88+
kv-cache-dtype: "fp8_e4m3"
89+
trust-remote-code: true
90+
attention-backend: "trtllm_mha"
91+
tensor-parallel-size: 8
92+
data-parallel-size: 1
93+
expert-parallel-size: 8
94+
mamba-scheduler-strategy: "no_buffer"
95+
mamba-track-interval: 2048
96+
mamba-ssm-dtype: "bfloat16"
97+
moe-runner-backend: "flashinfer_trtllm"
98+
disable-radix-cache: true
99+
max-running-requests: 1024
100+
mem-fraction-static: 0.8
101+
max-total-tokens: 128000
102+
chunked-prefill-size: 16384
103+
max-prefill-tokens: 16384
104+
context-length: 9236
105+
cuda-graph-max-bs: 320
106+
scheduler-recv-interval: 10
107+
decode-log-interval: 50
108+
stream-interval: 50
109+
disaggregation-mode: "prefill"
110+
disaggregation-transfer-backend: "mooncake"
111+
112+
decode:
113+
served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8"
114+
model-path: "/model/"
115+
random-seed: 42
116+
quantization: "fp8"
117+
kv-cache-dtype: "fp8_e4m3"
118+
trust-remote-code: true
119+
attention-backend: "trtllm_mha"
120+
tensor-parallel-size: 8
121+
data-parallel-size: 1
122+
expert-parallel-size: 8
123+
mamba-scheduler-strategy: "no_buffer"
124+
mamba-track-interval: 128
125+
mamba-ssm-dtype: "bfloat16"
126+
max-mamba-cache-size: 1024
127+
moe-runner-backend: "flashinfer_trtllm"
128+
129+
speculative-algorithm: "EAGLE"
130+
speculative-num-steps: 3
131+
speculative-eagle-topk: 1
132+
speculative-num-draft-tokens: 4
133+
134+
disable-radix-cache: true
135+
max-running-requests: 1024
136+
mem-fraction-static: 0.8
137+
max-total-tokens: 2200000
138+
chunked-prefill-size: 4096
139+
max-prefill-tokens: 16384
140+
context-length: 9236
141+
cuda-graph-max-bs: 320
142+
scheduler-recv-interval: 10
143+
decode-log-interval: 50
144+
stream-interval: 50
145+
disaggregation-mode: "decode"
146+
disaggregation-transfer-backend: "mooncake"
147+
148+
benchmark:
149+
type: "sa-bench"
150+
isl: 8192
151+
osl: 1024
152+
req_rate: "inf"
153+
num_prompts_mult: 20
154+
num_warmup_mult: 2
155+
random_range_ratio: 0.8
156+
concurrencies: "32x48x80"
157+
use_chat_template: true

0 commit comments

Comments
 (0)