Skip to content

Commit c1737ae

Browse files
seungrokjseungrokj
authored
Merge branch 'main' into amd/agentx_dsv4_sgl_mtp_0717
2 parents 43d99c5 + 5ba0b0b commit c1737ae

13 files changed

Lines changed: 677 additions & 98 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,22 @@ name: "svf-vllm-disagg-gb200-high-tpt-megamoe"
88
# 4096 with no CPU/NVMe offload.
99
#
1010
# Local deltas vs upstream:
11-
# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match
12-
# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh.
13-
# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to
11+
# * model.path uses the deepseek-v4-pro-mxfp4 alias from
12+
# runners/launch_gb200-nv.sh.
13+
# * model.container set to vllm/vllm-openai:v0.25.1 to
1414
# match nvidia-master.yaml image (which the launch script registers as
1515
# the alias key in srtslurm.yaml). Upstream variants ship either the
1616
# non-dynamo floating tag or a sha256 pin.
1717
# * slurm.time_limit + health_check set to 8h / 1440 attempts to
1818
# absorb cold-cache /mnt/numa1 model loads.
1919
model:
20-
path: "deepseek-v4-pro"
21-
container: "vllm/vllm-openai:v0.20.0-ubuntu2404"
20+
path: "deepseek-v4-pro-mxfp4"
21+
container: "vllm/vllm-openai:v0.25.1"
2222
precision: "fp4"
2323

2424
dynamo:
2525
install: true
26-
wheel: "1.2.0.dev20260426"
27-
28-
setup_script: vllm-container-deps.sh
26+
wheel: "1.3.0.dev1"
2927

3028
slurm:
3129
time_limit: "8:00:00"
@@ -48,11 +46,14 @@ infra:
4846

4947
frontend:
5048
type: dynamo
51-
enable_multiple_frontends: false
49+
enable_multiple_frontends: true
50+
env:
51+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5252
backend:
5353
type: vllm
5454
connector: null
5555
prefill_environment:
56+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5657
TILELANG_CLEANUP_TEMP_FILES: "1"
5758
VLLM_USE_NCCL_SYMM_MEM: "1"
5859
TORCH_SYMMMEM: "NVSHMEM"
@@ -70,6 +71,7 @@ backend:
7071
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
7172
NCCL_P2P_LEVEL: NVL
7273
decode_environment:
74+
DYN_REQUEST_PLANE_CODEC: "msgpack"
7375
TILELANG_CLEANUP_TEMP_FILES: "1"
7476
VLLM_USE_NCCL_SYMM_MEM: "1"
7577
TORCH_SYMMMEM: "NVSHMEM"
@@ -147,9 +149,9 @@ benchmark:
147149
identity:
148150
model:
149151
repo: "deepseek-ai/DeepSeek-V4-Pro"
150-
revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
152+
revision: "b5968e9190ef611bbf34a7229255be88a0e937c1"
151153
container:
152-
image: "vllm/vllm-openai:v0.20.0-ubuntu2404"
154+
image: "vllm/vllm-openai:v0.25.1"
153155
frameworks:
154-
dynamo: "1.2.0.dev20260426"
155-
vllm: "0.20.0"
156+
dynamo: "1.3.0.dev1"
157+
vllm: "0.25.1"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,22 @@ name: "svf-vllm-disagg-gb200-low-latency"
77
# dedicated NATS/etcd infra node. Single-concurrency point for low latency.
88
#
99
# Local deltas vs upstream:
10-
# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match
11-
# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh.
12-
# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to
10+
# * model.path uses the deepseek-v4-pro-mxfp4 alias from
11+
# runners/launch_gb200-nv.sh.
12+
# * model.container set to vllm/vllm-openai:v0.25.1 to
1313
# match nvidia-master.yaml image (which the launch script registers as
1414
# the alias key in srtslurm.yaml). Upstream variants ship either the
1515
# non-dynamo floating tag or a sha256 pin.
1616
# * slurm.time_limit + health_check set to 8h / 1440 attempts to
1717
# absorb cold-cache /mnt/numa1 model loads.
1818
model:
19-
path: "deepseek-v4-pro"
20-
container: "vllm/vllm-openai:v0.20.0-ubuntu2404"
19+
path: "deepseek-v4-pro-mxfp4"
20+
container: "vllm/vllm-openai:v0.25.1"
2121
precision: "fp4"
2222

2323
dynamo:
2424
install: true
25-
wheel: "1.2.0.dev20260426"
26-
27-
setup_script: vllm-container-deps.sh
25+
wheel: "1.3.0.dev1"
2826

2927
slurm:
3028
time_limit: "8:00:00"
@@ -47,11 +45,14 @@ infra:
4745

4846
frontend:
4947
type: dynamo
50-
enable_multiple_frontends: false
48+
enable_multiple_frontends: true
49+
env:
50+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5151
backend:
5252
type: vllm
5353
connector: null
5454
prefill_environment:
55+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5556
TILELANG_CLEANUP_TEMP_FILES: "1"
5657
VLLM_USE_NCCL_SYMM_MEM: "1"
5758
NCCL_CUMEM_ENABLE: "1"
@@ -68,6 +69,7 @@ backend:
6869
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
6970
NCCL_P2P_LEVEL: NVL
7071
decode_environment:
72+
DYN_REQUEST_PLANE_CODEC: "msgpack"
7173
TILELANG_CLEANUP_TEMP_FILES: "1"
7274
VLLM_USE_NCCL_SYMM_MEM: "1"
7375
NCCL_CUMEM_ENABLE: "1"
@@ -142,8 +144,11 @@ benchmark:
142144
custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
143145

144146
identity:
147+
model:
148+
repo: "deepseek-ai/DeepSeek-V4-Pro"
149+
revision: "b5968e9190ef611bbf34a7229255be88a0e937c1"
145150
container:
146-
image: "vllm/vllm-openai:v0.20.0-ubuntu2404"
151+
image: "vllm/vllm-openai:v0.25.1"
147152
frameworks:
148-
dynamo: "1.2.0.dev20260426"
149-
vllm: "0.20.0"
153+
dynamo: "1.3.0.dev1"
154+
vllm: "0.25.1"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,22 @@ name: "svf-vllm-disagg-gb200-low-middle-curve"
88
# 256 and 512.
99
#
1010
# Local deltas vs upstream:
11-
# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match
12-
# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh.
13-
# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to
11+
# * model.path uses the deepseek-v4-pro-mxfp4 alias from
12+
# runners/launch_gb200-nv.sh.
13+
# * model.container set to vllm/vllm-openai:v0.25.1 to
1414
# match nvidia-master.yaml image (which the launch script registers as
1515
# the alias key in srtslurm.yaml). Upstream variants ship either the
1616
# non-dynamo floating tag or a sha256 pin.
1717
# * slurm.time_limit + health_check set to 8h / 1440 attempts to
1818
# absorb cold-cache /mnt/numa1 model loads.
1919
model:
20-
path: "deepseek-v4-pro"
21-
container: "vllm/vllm-openai:v0.20.0-ubuntu2404"
20+
path: "deepseek-v4-pro-mxfp4"
21+
container: "vllm/vllm-openai:v0.25.1"
2222
precision: "fp4"
2323

2424
dynamo:
2525
install: true
26-
wheel: "1.2.0.dev20260426"
27-
28-
setup_script: vllm-container-deps.sh
26+
wheel: "1.3.0.dev1"
2927

3028
slurm:
3129
time_limit: "8:00:00"
@@ -48,11 +46,14 @@ infra:
4846

4947
frontend:
5048
type: dynamo
51-
enable_multiple_frontends: false
49+
enable_multiple_frontends: true
50+
env:
51+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5252
backend:
5353
type: vllm
5454
connector: null
5555
prefill_environment:
56+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5657
TILELANG_CLEANUP_TEMP_FILES: "1"
5758
VLLM_USE_NCCL_SYMM_MEM: "1"
5859
NCCL_CUMEM_ENABLE: "1"
@@ -69,6 +70,7 @@ backend:
6970
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
7071
NCCL_P2P_LEVEL: NVL
7172
decode_environment:
73+
DYN_REQUEST_PLANE_CODEC: "msgpack"
7274
TILELANG_CLEANUP_TEMP_FILES: "1"
7375
VLLM_USE_NCCL_SYMM_MEM: "1"
7476
NCCL_CUMEM_ENABLE: "1"
@@ -129,7 +131,7 @@ backend:
129131
no-enable-flashinfer-autotune: true
130132
block-size: 256
131133
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
132-
gpu-memory-utilization: 0.9
134+
gpu-memory-utilization: 0.85
133135
stream-interval: 50
134136
no-disable-hybrid-kv-cache-manager: true
135137
enable-sleep-mode: true
@@ -144,8 +146,11 @@ benchmark:
144146
custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
145147

146148
identity:
149+
model:
150+
repo: "deepseek-ai/DeepSeek-V4-Pro"
151+
revision: "b5968e9190ef611bbf34a7229255be88a0e937c1"
147152
container:
148-
image: "vllm/vllm-openai:v0.20.0-ubuntu2404"
153+
image: "vllm/vllm-openai:v0.25.1"
149154
frameworks:
150-
dynamo: "1.2.0.dev20260426"
151-
vllm: "0.20.0"
155+
dynamo: "1.3.0.dev1"
156+
vllm: "0.25.1"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,22 @@ name: "svf-vllm-disagg-gb200-max-tpt-megamoe"
88
# 4096 with no CPU/NVMe offload.
99
#
1010
# Local deltas vs upstream:
11-
# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match
12-
# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh.
13-
# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to
11+
# * model.path uses the deepseek-v4-pro-mxfp4 alias from
12+
# runners/launch_gb200-nv.sh.
13+
# * model.container set to vllm/vllm-openai:v0.25.1 to
1414
# match nvidia-master.yaml image (which the launch script registers as
1515
# the alias key in srtslurm.yaml). Upstream variants ship either the
1616
# non-dynamo floating tag or a sha256 pin.
1717
# * slurm.time_limit + health_check set to 8h / 1440 attempts to
1818
# absorb cold-cache /mnt/numa1 model loads.
1919
model:
20-
path: "deepseek-v4-pro"
21-
container: "vllm/vllm-openai:v0.20.0-ubuntu2404"
20+
path: "deepseek-v4-pro-mxfp4"
21+
container: "vllm/vllm-openai:v0.25.1"
2222
precision: "fp4"
2323

2424
dynamo:
2525
install: true
26-
wheel: "1.2.0.dev20260426"
27-
28-
setup_script: vllm-container-deps.sh
26+
wheel: "1.3.0.dev1"
2927

3028
slurm:
3129
time_limit: "8:00:00"
@@ -48,11 +46,14 @@ infra:
4846

4947
frontend:
5048
type: dynamo
51-
enable_multiple_frontends: false
49+
enable_multiple_frontends: true
50+
env:
51+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5252
backend:
5353
type: vllm
5454
connector: null
5555
prefill_environment:
56+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5657
TILELANG_CLEANUP_TEMP_FILES: "1"
5758
VLLM_USE_NCCL_SYMM_MEM: "1"
5859
TORCH_SYMMMEM: "NVSHMEM"
@@ -70,6 +71,7 @@ backend:
7071
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
7172
NCCL_P2P_LEVEL: NVL
7273
decode_environment:
74+
DYN_REQUEST_PLANE_CODEC: "msgpack"
7375
TILELANG_CLEANUP_TEMP_FILES: "1"
7476
VLLM_USE_NCCL_SYMM_MEM: "1"
7577
TORCH_SYMMMEM: "NVSHMEM"
@@ -147,9 +149,9 @@ benchmark:
147149
identity:
148150
model:
149151
repo: "deepseek-ai/DeepSeek-V4-Pro"
150-
revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
152+
revision: "b5968e9190ef611bbf34a7229255be88a0e937c1"
151153
container:
152-
image: "vllm/vllm-openai:v0.20.0-ubuntu2404"
154+
image: "vllm/vllm-openai:v0.25.1"
153155
frameworks:
154-
dynamo: "1.2.0.dev20260426"
155-
vllm: "0.20.0"
156+
dynamo: "1.3.0.dev1"
157+
vllm: "0.25.1"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,22 @@ name: "svf-vllm-disagg-gb200-mid-curve-megamoe"
88
# 256/512/1024 with no CPU/NVMe offload.
99
#
1010
# Local deltas vs upstream:
11-
# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match
12-
# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh.
13-
# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to
11+
# * model.path uses the deepseek-v4-pro-mxfp4 alias from
12+
# runners/launch_gb200-nv.sh.
13+
# * model.container set to vllm/vllm-openai:v0.25.1 to
1414
# match nvidia-master.yaml image (which the launch script registers as
1515
# the alias key in srtslurm.yaml). Upstream variants ship either the
1616
# non-dynamo floating tag or a sha256 pin.
1717
# * slurm.time_limit + health_check set to 8h / 1440 attempts to
1818
# absorb cold-cache /mnt/numa1 model loads.
1919
model:
20-
path: "deepseek-v4-pro"
21-
container: "vllm/vllm-openai:v0.20.0-ubuntu2404"
20+
path: "deepseek-v4-pro-mxfp4"
21+
container: "vllm/vllm-openai:v0.25.1"
2222
precision: "fp4"
2323

2424
dynamo:
2525
install: true
26-
wheel: "1.2.0.dev20260426"
27-
28-
setup_script: vllm-container-deps.sh
26+
wheel: "1.3.0.dev1"
2927

3028
slurm:
3129
time_limit: "8:00:00"
@@ -48,11 +46,14 @@ infra:
4846

4947
frontend:
5048
type: dynamo
51-
enable_multiple_frontends: false
49+
enable_multiple_frontends: true
50+
env:
51+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5252
backend:
5353
type: vllm
5454
connector: null
5555
prefill_environment:
56+
DYN_REQUEST_PLANE_CODEC: "msgpack"
5657
TILELANG_CLEANUP_TEMP_FILES: "1"
5758
VLLM_USE_NCCL_SYMM_MEM: "1"
5859
TORCH_SYMMMEM: "NVSHMEM"
@@ -70,6 +71,7 @@ backend:
7071
UCX_CUDA_IPC_ENABLE_MNNVL: "y"
7172
NCCL_P2P_LEVEL: NVL
7273
decode_environment:
74+
DYN_REQUEST_PLANE_CODEC: "msgpack"
7375
TILELANG_CLEANUP_TEMP_FILES: "1"
7476
VLLM_USE_NCCL_SYMM_MEM: "1"
7577
TORCH_SYMMMEM: "NVSHMEM"
@@ -147,9 +149,9 @@ benchmark:
147149
identity:
148150
model:
149151
repo: "deepseek-ai/DeepSeek-V4-Pro"
150-
revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
152+
revision: "b5968e9190ef611bbf34a7229255be88a0e937c1"
151153
container:
152-
image: "vllm/vllm-openai:v0.20.0-ubuntu2404"
154+
image: "vllm/vllm-openai:v0.25.1"
153155
frameworks:
154-
dynamo: "1.2.0.dev20260426"
155-
vllm: "0.20.0"
156+
dynamo: "1.3.0.dev1"
157+
vllm: "0.25.1"

0 commit comments

Comments
 (0)