Skip to content

Commit cb438b7

Browse files
ivaniumivanium
committed
agetnx: gb300 pd configs
1 parent 95f5dd8 commit cb438b7

8 files changed

Lines changed: 1032 additions & 11 deletions

File tree

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: "svf-vllm-agg-gb300-tp4-agentic"
2+
3+
# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe on one four-GPU GB300
4+
# node. The worker serves both prefill and decode with TP4.
5+
6+
model:
7+
path: "deepseek-v4-pro"
8+
container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
9+
precision: "fp4"
10+
11+
identity:
12+
model:
13+
repo: "deepseek-ai/DeepSeek-V4-Pro"
14+
container:
15+
image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
16+
frameworks:
17+
dynamo: "1.2.1"
18+
vllm: "0.17.2rc1.dev3675+gc188b96eb"
19+
20+
dynamo:
21+
hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0"
22+
install: true
23+
24+
setup_script: vllm-container-deps.sh
25+
26+
slurm:
27+
time_limit: "8:00:00"
28+
29+
health_check:
30+
max_attempts: 2160
31+
interval_seconds: 10
32+
33+
resources:
34+
gpu_type: "gb300"
35+
gpus_per_node: 4
36+
agg_nodes: 1
37+
agg_workers: 1
38+
gpus_per_agg: 4
39+
40+
infra:
41+
etcd_nats_dedicated_node: false
42+
nats_max_payload_mb: 32
43+
44+
frontend:
45+
type: dynamo
46+
enable_multiple_frontends: false
47+
args:
48+
router-mode: "kv"
49+
router-reset-states: true
50+
router-temperature: 0.0
51+
router-queue-threshold: 65536
52+
active-decode-blocks-threshold: "None"
53+
active-prefill-tokens-threshold: "None"
54+
active-prefill-tokens-threshold-frac: "None"
55+
tokenizer: "fastokens"
56+
57+
backend:
58+
type: vllm
59+
connector: null
60+
mooncake_kv_store:
61+
store_config:
62+
metadata_server: "P2PHANDSHAKE"
63+
global_segment_size: "150GB"
64+
local_buffer_size: "4GB"
65+
protocol: "rdma"
66+
device_name: ""
67+
mode: "embedded"
68+
enable_offload: false
69+
aggregated_environment:
70+
DYN_REQUEST_PLANE: "tcp"
71+
ETCD_LEASE_TTL: "120"
72+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
73+
VLLM_RPC_TIMEOUT: "600000"
74+
TILELANG_CLEANUP_TEMP_FILES: "1"
75+
VLLM_USE_NCCL_SYMM_MEM: "1"
76+
TORCH_SYMMMEM: "NVSHMEM"
77+
NCCL_CUMEM_ENABLE: "1"
78+
NCCL_MNNVL_ENABLE: "1"
79+
NCCL_NVLS_ENABLE: "1"
80+
VLLM_SERVER_DEV_MODE: "1"
81+
VLLM_USE_V2_MODEL_RUNNER: "1"
82+
VLLM_USE_RUST_FRONTEND: "1"
83+
VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
84+
VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
85+
VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65"
86+
DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}"
87+
VLLM_DSV4_MEGA_FP8_COMBINE: "1"
88+
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
89+
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
90+
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
91+
UCX_MEMTYPE_CACHE: "n"
92+
UCX_MEMTYPE_REG_WHOLE: "n"
93+
UCX_RCACHE_MAX_UNRELEASED: "1024"
94+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
95+
NCCL_P2P_LEVEL: NVL
96+
MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
97+
98+
vllm_config:
99+
aggregated:
100+
kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
101+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
102+
kv-cache-dtype: "fp8"
103+
tensor-parallel-size: 4
104+
pipeline-parallel-size: 1
105+
enable-cumem-allocator: true
106+
numa-bind: true
107+
attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
108+
max-model-len: 1048576
109+
max-num-seqs: 32
110+
max-num-batched-tokens: 32768
111+
trust-remote-code: true
112+
no-enable-flashinfer-autotune: true
113+
block-size: 256
114+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
115+
max-cudagraph-capture-size: 32
116+
gpu-memory-utilization: 0.95
117+
stream-interval: 10
118+
no-disable-hybrid-kv-cache-manager: true
119+
tokenizer-mode: deepseek_v4
120+
121+
sbatch_directives:
122+
cpus-per-task: "72"
123+
124+
srun_options:
125+
container-remap-root: ""
126+
127+
benchmark:
128+
type: custom
129+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
130+
env:
131+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
132+
RESULT_DIR: /logs/agentic
133+
PORT: "8000"
134+
IS_MULTINODE: "false"
135+
TP: "4"
136+
EP_SIZE: "1"
137+
DP_ATTENTION: "false"
138+
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
139+
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
140+
AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
141+
HF_HUB_CACHE: "/hf_hub_cache"
142+
WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

‎benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: "svf-vllm-agg-gb300-tp8-agentic"
2+
3+
# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe spanning two four-GPU
4+
# GB300 nodes. The worker serves both prefill and decode with TP8.
5+
6+
model:
7+
path: "deepseek-v4-pro"
8+
container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
9+
precision: "fp4"
10+
11+
identity:
12+
model:
13+
repo: "deepseek-ai/DeepSeek-V4-Pro"
14+
container:
15+
image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96"
16+
frameworks:
17+
dynamo: "1.2.1"
18+
vllm: "0.17.2rc1.dev3675+gc188b96eb"
19+
20+
dynamo:
21+
hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0"
22+
install: true
23+
24+
setup_script: vllm-container-deps.sh
25+
26+
slurm:
27+
time_limit: "8:00:00"
28+
29+
health_check:
30+
max_attempts: 2160
31+
interval_seconds: 10
32+
33+
resources:
34+
gpu_type: "gb300"
35+
gpus_per_node: 4
36+
agg_nodes: 2
37+
agg_workers: 1
38+
gpus_per_agg: 8
39+
40+
infra:
41+
etcd_nats_dedicated_node: false
42+
nats_max_payload_mb: 32
43+
44+
frontend:
45+
type: dynamo
46+
enable_multiple_frontends: false
47+
args:
48+
router-mode: "kv"
49+
router-reset-states: true
50+
router-temperature: 0.0
51+
router-queue-threshold: 65536
52+
active-decode-blocks-threshold: "None"
53+
active-prefill-tokens-threshold: "None"
54+
active-prefill-tokens-threshold-frac: "None"
55+
tokenizer: "fastokens"
56+
57+
backend:
58+
type: vllm
59+
connector: null
60+
mooncake_kv_store:
61+
store_config:
62+
metadata_server: "P2PHANDSHAKE"
63+
global_segment_size: "150GB"
64+
local_buffer_size: "4GB"
65+
protocol: "rdma"
66+
device_name: ""
67+
mode: "embedded"
68+
enable_offload: false
69+
aggregated_environment:
70+
DYN_REQUEST_PLANE: "tcp"
71+
ETCD_LEASE_TTL: "120"
72+
VLLM_ENGINE_READY_TIMEOUT_S: "3600"
73+
VLLM_RPC_TIMEOUT: "600000"
74+
TILELANG_CLEANUP_TEMP_FILES: "1"
75+
VLLM_USE_NCCL_SYMM_MEM: "1"
76+
TORCH_SYMMMEM: "NVSHMEM"
77+
NCCL_CUMEM_ENABLE: "1"
78+
NCCL_MNNVL_ENABLE: "1"
79+
NCCL_NVLS_ENABLE: "1"
80+
VLLM_SERVER_DEV_MODE: "1"
81+
VLLM_USE_V2_MODEL_RUNNER: "1"
82+
VLLM_USE_RUST_FRONTEND: "1"
83+
VLLM_MOONCAKE_LOAD_RECV_THREADS: "4"
84+
VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
85+
VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65"
86+
DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}"
87+
VLLM_DSV4_MEGA_FP8_COMBINE: "1"
88+
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
89+
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
90+
VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
91+
UCX_MEMTYPE_CACHE: "n"
92+
UCX_MEMTYPE_REG_WHOLE: "n"
93+
UCX_RCACHE_MAX_UNRELEASED: "1024"
94+
UCX_TLS: "cuda_copy,cuda_ipc,tcp"
95+
NCCL_P2P_LEVEL: NVL
96+
MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
97+
98+
vllm_config:
99+
aggregated:
100+
kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}'
101+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
102+
kv-cache-dtype: "fp8"
103+
tensor-parallel-size: 8
104+
pipeline-parallel-size: 1
105+
enable-cumem-allocator: true
106+
numa-bind: true
107+
attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
108+
max-model-len: 1048576
109+
max-num-seqs: 32
110+
max-num-batched-tokens: 32768
111+
trust-remote-code: true
112+
no-enable-flashinfer-autotune: true
113+
block-size: 256
114+
compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
115+
max-cudagraph-capture-size: 32
116+
gpu-memory-utilization: 0.95
117+
stream-interval: 10
118+
no-disable-hybrid-kv-cache-manager: true
119+
tokenizer-mode: deepseek_v4
120+
121+
sbatch_directives:
122+
cpus-per-task: "72"
123+
124+
srun_options:
125+
container-remap-root: ""
126+
127+
benchmark:
128+
type: custom
129+
command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
130+
env:
131+
INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
132+
RESULT_DIR: /logs/agentic
133+
PORT: "8000"
134+
IS_MULTINODE: "false"
135+
TP: "8"
136+
EP_SIZE: "1"
137+
DP_ATTENTION: "false"
138+
AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
139+
AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
140+
AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
141+
HF_HUB_CACHE: "/hf_hub_cache"
142+
WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"

0 commit comments

Comments
 (0)