From 4bd78730fcb5ae3cecfb13e30b64fd8964625080 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Tue, 21 Jul 2026 22:27:06 +0000 Subject: [PATCH 1/5] feat(agentx): add DeepSeek-V4 GB200 Dynamo recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the aggregate TP8 and four disaggregated DP/EP topologies validated by PR #2260. Use current NVIDIA/srt-slurm main for upstream per-node DP, health-count, and Mooncake fixes while retaining the still-required multi-node TP port patch. Configure the TP8 path to use FlashInfer all-reduce. Limit this validation sweep to TP8 c16 and representative 1P1D DEP8/DEP8 c256. 中文：新增 DeepSeek-V4 GB200 Dynamo AgentX 聚合式 TP8 配置及四种分离式 DP/EP 拓扑，均已通过 PR #2260 验证。改用 NVIDIA/srt-slurm 最新 main 分支以复用已上游的单节点内 DP 启动、健康检查计数和 Mooncake 修复，仅保留多节点 TP 仍需的端口补丁；同时为 TP8 启用 FlashInfer all-reduce。本次验证扫描仅运行 TP8 c16 和代表性的 1P1D DEP8/DEP8 c256。 --- benchmarks/benchmark_lib.sh | 6 +- .../agentic/agg-gb200-tp8-agentic.yaml | 155 ++++++++++++++ .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 200 +++++++++++++++++ .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 200 +++++++++++++++++ .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 200 +++++++++++++++++ .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 201 ++++++++++++++++++ configs/nvidia-master.yaml | 146 +++++++++++++ perf-changelog.yaml | 10 + runners/launch_gb200-nv.sh | 43 ++-- .../srt-slurm-vllm-multinode-tp-port.patch | 17 ++ 10 files changed, 1158 insertions(+), 20 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml create mode 100644 runners/patches/srt-slurm-vllm-multinode-tp-port.patch diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9a01f63b4b..a318b074ce 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml new file mode 100644 index 0000000000..0df9ca0c44 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -0,0 +1,155 @@ +name: "svf-vllm-agg-gb200-tp8-agentic" + +# AgentX low-concurrency aggregate recipe for DeepSeek-V4-Pro on GB200. One +# TP8 vLLM worker spans two four-GPU nodes and serves both prefill and decode. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "600" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + # Off: the c188b96 container's CUDA-graph capture path enters vLLM's + # nccl_symm_mem_context without set_graph_pool_id, so capture asserts + # "graph_pool_id is not set under graph capture". Re-enable once this recipe + # moves to a container built from a vLLM with the cuda_graph.py fix. + VLLM_USE_NCCL_SYMM_MEM: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + enable-cumem-allocator: true + # numa-bind disabled: srt-slurm's cpus-per-task/cgroup already constrains + # CPU affinity, so vLLM's auto-NUMA detection bails and --numa-bind hard- + # errors ("could not detect the GPU-to-NUMA topology"). Re-enable only with + # an explicit --numa-bind-nodes mapping for this cluster. + # numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + # AgentX session fan-out can exceed the configured live-session + # concurrency; retain scheduler/graph headroom for subagent bursts. + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + kv-cache-memory: 37580963840 # 35 GiB + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # One aggregate worker owns all eight GPUs. Use the aggregate accounting + # path instead of summing independent prefill and decode allocations. + # Aggregate (IS_MULTINODE=false) uses the single-node topology labels below + # directly (num_gpus = TP), so they are set here rather than CI-injected. + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..ac2a0bd302 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,200 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep12-agentic" + +# One DEP8 prefill worker feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml new file mode 100644 index 0000000000..efa076c375 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -0,0 +1,200 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-agentic" + +# One DEP8 prefill worker feeding one DEP8 decode worker, with attention DP8 / +# expert EP8 and hybrid load balancing on both roles. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 64 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..92d9bc4601 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,200 @@ +name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-agentic" + +# Two DEP8 prefill workers feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 192 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml new file mode 100644 index 0000000000..fcfe6a99bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -0,0 +1,201 @@ +name: "svf-vllm-disagg-gb200-3p1d-dep8-dep16-agentic" + +# Three DEP8 prefill workers feeding one DEP16 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. During the ~380s + # weight load the process is CPU-saturated and the Rust keep-alive task + # cannot renew in time, so etcd revokes the lease and worker registration + # later fails with "requested lease not found" (workers never go ready -> + # health check times out at 2h). 600s covers the weight load + cudagraph + # stalls; longer TTL only delays dead-worker cleanup, which the health + # check / fail-fast monitor handle independently. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 214 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 160 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 160 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a167f49270..535dab74c7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7881,6 +7881,152 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb200-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [384, 512] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml" + decode: + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [640, 720, 768] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml" + decode: + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [800, 960, 1024, 1280] + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 335245a6c3..3fad3da11d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5060,3 +5060,13 @@ - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 + description: + - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 and four disaggregated recipe/config definitions; limit this validation sweep to aggregate TP8 at conc [16] and representative 1P/1D DEP8/DEP8 at conc [256]." + - "Retain the additional tuned PD definitions for future sweeps: 1P/1D DEP8/DEP12 at [384,512], 2P/1D DEP8/DEP12 at [640,720,768], and 3P/1D DEP8/DEP16 at [800,960,1024,1280]." + - "Use NVIDIA/srt-slurm main with its upstream per-node DP, Dynamo health-count, and Mooncake master fixes; retain the unmerged multi-node TP VLLM_PORT fix. TP8 uses FlashInfer all-reduce and disables vLLM custom all-reduce." + - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index a5f917da09..2867d38c65 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -156,11 +156,15 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # The FP4 checkpoint is staged on compute-visible Lustre. The former - # /mnt/numa1 path is no longer present on watchtower compute nodes; - # the lowercase Lustre sibling is the FP8 checkpoint, so keep the - # NVFP4 path explicit here. - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + # AgentX was tuned against the canonical DeepSeek-V4-Pro + # checkpoint, which is staged on compute-visible Lustre. + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro/" + else + # Existing fixed-sequence GB200 recipes use the NVIDIA ModelOpt + # NVFP4 checkpoint. + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + fi export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" @@ -293,21 +297,22 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses the same pinned cquil11/srt-slurm-nv commit as - # launch_gb300-nv.sh — everything the agentic recipes need is there: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) - # - srtctl apply --no-preflight (model path /mnt/numa1 is compute-node - # local NVMe, invisible to the login-node runner) - # - benchmark_stage srun_options propagation (container-remap-root - # must reach the agentic_srt.sh srun) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout de59739b172e507e15ebf145bfe305f606e82fbf - mkdir -p recipes/vllm/deepseek-v4/agentic + # Agentic recipes use NVIDIA/srt-slurm:main. The per-node DP launcher, + # matching Dynamo health counts, and Mooncake master compatibility are all + # upstream; keep only the still-unmerged multi-node TP port fix below. + git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + + # NVIDIA/srt-slurm#243 gives each srt process a private VLLM_PORT range, but + # a multi-node TP process still passes that one range to all of its local + # vLLM children. Unset it only for multi-node non-DP workers, matching the + # still-unmerged NVIDIA/srt-slurm commit de1a4f0257dae5bf871881dc4696e35389c37483. + # Keep the upstream allocation for per-node DP and co-located endpoints. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-vllm-multinode-tp-port.patch" || exit 1 + + mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + recipes/vllm/deepseek-v4/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" diff --git a/runners/patches/srt-slurm-vllm-multinode-tp-port.patch b/runners/patches/srt-slurm-vllm-multinode-tp-port.patch new file mode 100644 index 0000000000..1e15540511 --- /dev/null +++ b/runners/patches/srt-slurm-vllm-multinode-tp-port.patch @@ -0,0 +1,17 @@ +diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py +--- a/src/srtctl/backends/vllm.py ++++ b/src/srtctl/backends/vllm.py +@@ -829,4 +829,12 @@ class VLLMProtocol: + # Add all config flags from vllm_config + cmd.extend(_config_to_cli_args(config)) +- ++ ++ if is_multi_node and not is_dp_mode: ++ # Avoid letting an inherited/container VLLM_PORT seed vLLM's ++ # internal cross-node message-queue port allocator. All local TP ++ # worker subprocesses inherit that same fixed base and can race ++ # when binding their ZMQ sockets; leaving it unset lets the kernel ++ # choose free ports for those internal queues. ++ cmd = ["env", "-u", "VLLM_PORT"] + cmd ++ + return cmd From 8ceef6a6b4f3c0a710a07316d90951e867460125 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 22 Jul 2026 09:47:47 +0000 Subject: [PATCH 2/5] chore(agentx): trim GB200 validation sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep only the aggregate TP8 c16 and 3P1D DEP8/DEP16 c1536 validation arms. Remove the three unused PD recipes and master-config entries, and point the changelog at PR #2300. 中文：验证扫描仅保留聚合式 TP8 c16 和 3P1D DEP8/DEP16 c1536 两组配置；移除三份未使用的 PD 配方及对应主配置，并将性能变更日志链接更新为 PR #2300。 --- .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 200 ------------------ .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 200 ------------------ .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 200 ------------------ configs/nvidia-master.yaml | 89 +------- perf-changelog.yaml | 7 +- 5 files changed, 4 insertions(+), 692 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml deleted file mode 100644 index ac2a0bd302..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ /dev/null @@ -1,200 +0,0 @@ -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep12-agentic" - -# One DEP8 prefill worker feeding one DEP12 decode worker, with attention / -# expert parallelism matching each role and hybrid load balancing. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -dynamo: - hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 12 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local - # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" - mode: "embedded" - enable_offload: false - prefill_environment: - # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall - # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd - # co-located with a worker) then revokes every worker's lease at once, so - # registration fails with "requested lease not found" and workers never go - # ready (health check times out at 2h). 600s rides through transient etcd - # stalls; longer TTL only delays dead-worker cleanup, handled independently. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - decode_environment: - # See prefill_environment: raise dynamo's primary etcd lease TTL from the - # 10s default so the lease survives the CPU-saturated cold start. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml deleted file mode 100644 index efa076c375..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ /dev/null @@ -1,200 +0,0 @@ -name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-agentic" - -# One DEP8 prefill worker feeding one DEP8 decode worker, with attention DP8 / -# expert EP8 and hybrid load balancing on both roles. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -dynamo: - hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local - # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" - mode: "embedded" - enable_offload: false - prefill_environment: - # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall - # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd - # co-located with a worker) then revokes every worker's lease at once, so - # registration fails with "requested lease not found" and workers never go - # ready (health check times out at 2h). 600s rides through transient etcd - # stalls; longer TTL only delays dead-worker cleanup, handled independently. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - decode_environment: - # See prefill_environment: raise dynamo's primary etcd lease TTL from the - # 10s default so the lease survives the CPU-saturated cold start. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 64 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml deleted file mode 100644 index 92d9bc4601..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ /dev/null @@ -1,200 +0,0 @@ -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-agentic" - -# Two DEP8 prefill workers feeding one DEP12 decode worker, with attention / -# expert parallelism matching each role and hybrid load balancing. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -dynamo: - hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 720 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 12 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local - # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" - mode: "embedded" - enable_offload: false - prefill_environment: - # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall - # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd - # co-located with a worker) then revokes every worker's lease at once, so - # registration fails with "requested lease not found" and workers never go - # ready (health check times out at 2h). 600s rides through transient etcd - # stalls; longer TTL only delays dead-worker cleanup, handled independently. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - decode_environment: - # See prefill_environment: raise dynamo's primary etcd lease TTL from the - # 10s default so the lease survives the CPU-saturated cold start. - ETCD_LEASE_TTL: "600" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_USE_NCCL_SYMM_MEM: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - # MC_STORE_CLIENT_METRIC: "1" - # MC_STORE_CLIENT_METRIC_INTERVAL: "5" - # MC_TE_METRIC: "0" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 192 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 12 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enable-cumem-allocator: true - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 535dab74c7..2d02ff92d4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7911,93 +7911,6 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-agg: ep: 1 dp-attn: false -dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [384, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - -dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [640, 720, 768] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 model: deepseek-ai/DeepSeek-V4-Pro @@ -8013,7 +7926,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: agentic-coding: - search-space: - spec-decoding: none - conc-list: [800, 960, 1024, 1280] + conc-list: [1536] prefill: num-worker: 3 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3fad3da11d..c8e94d61b9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5063,10 +5063,9 @@ - config-keys: - dsv4-fp4-gb200-dynamo-vllm-agentic-agg - - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 + - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 and four disaggregated recipe/config definitions; limit this validation sweep to aggregate TP8 at conc [16] and representative 1P/1D DEP8/DEP8 at conc [256]." - - "Retain the additional tuned PD definitions for future sweeps: 1P/1D DEP8/DEP12 at [384,512], 2P/1D DEP8/DEP12 at [640,720,768], and 3P/1D DEP8/DEP16 at [800,960,1024,1280]." + - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [16] and disaggregated 3P/1D DEP8/DEP16 at conc [1536]." - "Use NVIDIA/srt-slurm main with its upstream per-node DP, Dynamo health-count, and Mooncake master fixes; retain the unmerged multi-node TP VLLM_PORT fix. TP8 uses FlashInfer all-reduce and disables vLLM custom all-reduce." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2300 From fc5a15bf3d2c064f421415bafe86483227b5916a Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 22 Jul 2026 09:55:40 +0000 Subject: [PATCH 3/5] fix(agentx): rely on upstream VLLM_PORT handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the redundant local srt-slurm patch now that main unsets VLLM_PORT for multi-node vLLM endpoints. 中文：依赖上游的 VLLM_PORT 处理。当前 srt-slurm main 已会为跨节点 vLLM 端点取消设置 VLLM_PORT，因此移除冗余的本地补丁。 --- perf-changelog.yaml | 2 +- runners/launch_gb200-nv.sh | 11 ++--------- .../srt-slurm-vllm-multinode-tp-port.patch | 17 ----------------- 3 files changed, 3 insertions(+), 27 deletions(-) delete mode 100644 runners/patches/srt-slurm-vllm-multinode-tp-port.patch diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c8e94d61b9..10a6a2749b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5066,6 +5066,6 @@ - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [16] and disaggregated 3P/1D DEP8/DEP16 at conc [1536]." - - "Use NVIDIA/srt-slurm main with its upstream per-node DP, Dynamo health-count, and Mooncake master fixes; retain the unmerged multi-node TP VLLM_PORT fix. TP8 uses FlashInfer all-reduce and disables vLLM custom all-reduce." + - "Use NVIDIA/srt-slurm main with its upstream per-node DP, multi-node TP VLLM_PORT, Dynamo health-count, and Mooncake master fixes. TP8 uses FlashInfer all-reduce and disables vLLM custom all-reduce." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2300 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 2867d38c65..458dec6387 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -298,18 +298,11 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then # Agentic recipes use NVIDIA/srt-slurm:main. The per-node DP launcher, - # matching Dynamo health counts, and Mooncake master compatibility are all - # upstream; keep only the still-unmerged multi-node TP port fix below. + # matching Dynamo health counts, multi-node TP VLLM_PORT handling, and + # Mooncake master compatibility are all upstream. git clone --branch main --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - # NVIDIA/srt-slurm#243 gives each srt process a private VLLM_PORT range, but - # a multi-node TP process still passes that one range to all of its local - # vLLM children. Unset it only for multi-node non-DP workers, matching the - # still-unmerged NVIDIA/srt-slurm commit de1a4f0257dae5bf871881dc4696e35389c37483. - # Keep the upstream allocation for per-node DP and co-located endpoints. - git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-vllm-multinode-tp-port.patch" || exit 1 - mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic || exit 1 diff --git a/runners/patches/srt-slurm-vllm-multinode-tp-port.patch b/runners/patches/srt-slurm-vllm-multinode-tp-port.patch deleted file mode 100644 index 1e15540511..0000000000 --- a/runners/patches/srt-slurm-vllm-multinode-tp-port.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py ---- a/src/srtctl/backends/vllm.py -+++ b/src/srtctl/backends/vllm.py -@@ -829,4 +829,12 @@ class VLLMProtocol: - # Add all config flags from vllm_config - cmd.extend(_config_to_cli_args(config)) -- -+ -+ if is_multi_node and not is_dp_mode: -+ # Avoid letting an inherited/container VLLM_PORT seed vLLM's -+ # internal cross-node message-queue port allocator. All local TP -+ # worker subprocesses inherit that same fixed base and can race -+ # when binding their ZMQ sockets; leaving it unset lets the kernel -+ # choose free ports for those internal queues. -+ cmd = ["env", "-u", "VLLM_PORT"] + cmd -+ - return cmd From 8248e47370c0bd001470b361ecb5d9ebe0ee83d2 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Wed, 22 Jul 2026 20:36:35 +0000 Subject: [PATCH 4/5] chore(agentx): update replacement PR link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the performance changelog at the upstream-hosted replacement PR. 中文：更新替代 PR 链接。将性能变更日志指向由上游仓库分支创建的新 PR。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 10a6a2749b..c88a5cf965 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5068,4 +5068,4 @@ - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [16] and disaggregated 3P/1D DEP8/DEP16 at conc [1536]." - "Use NVIDIA/srt-slurm main with its upstream per-node DP, multi-node TP VLLM_PORT, Dynamo health-count, and Mooncake master fixes. TP8 uses FlashInfer all-reduce and disables vLLM custom all-reduce." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2300 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302 From 99a28ab427a035a83a6738791d91449e4614b847 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Thu, 23 Jul 2026 02:55:07 +0000 Subject: [PATCH 5/5] fix(agentx): keep Dynamo discovery alive during cold start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply ETCD_LEASE_TTL=7200 globally in both GB200 AgentX recipes so the frontend and all workers retain discovery leases through the two-hour health window. 中文：在两个 GB200 AgentX 配方中全局设置 ETCD_LEASE_TTL=7200，确保前端及所有 worker 在最长两小时的健康检查窗口内维持 Dynamo 服务发现租约。 --- .../agentic/agg-gb200-tp8-agentic.yaml | 6 +++++- .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 16 +++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 0df9ca0c44..f70a713f7d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -14,6 +14,11 @@ dynamo: setup_script: vllm-container-deps.sh +environment: + # The frontend shares Grace CPU capacity with the long TP8 cold start. Keep + # Dynamo discovery alive until the outer two-hour health check expires. + ETCD_LEASE_TTL: "7200" + slurm: time_limit: "8:00:00" @@ -59,7 +64,6 @@ backend: enable_offload: false aggregated_environment: DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "600" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index fcfe6a99bf..024bb22140 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -14,6 +14,11 @@ dynamo: setup_script: vllm-container-deps.sh +environment: + # Apply the lease to the frontend as well as every prefill/decode worker so + # discovery survives the long CPU-saturated cold start. + ETCD_LEASE_TTL: "7200" + slurm: time_limit: "8:00:00" @@ -65,14 +70,6 @@ backend: mode: "embedded" enable_offload: false prefill_environment: - # dynamo's primary etcd lease defaults to a 10s TTL. During the ~380s - # weight load the process is CPU-saturated and the Rust keep-alive task - # cannot renew in time, so etcd revokes the lease and worker registration - # later fails with "requested lease not found" (workers never go ready -> - # health check times out at 2h). 600s covers the weight load + cudagraph - # stalls; longer TTL only delays dead-worker cleanup, which the health - # check / fail-fast monitor handle independently. - ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" @@ -101,9 +98,6 @@ backend: # MC_STORE_CLIENT_METRIC_INTERVAL: "5" # MC_TE_METRIC: "0" decode_environment: - # See prefill_environment: raise dynamo's primary etcd lease TTL from the - # 10s default so the lease survives the CPU-saturated cold start. - ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1"