From 6b06ade770d2aff3bc0cf15a9cb67128b215e3e5 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 08:24:45 +0000 Subject: [PATCH 01/22] agentx: dsv4 gb200 disagg --- .../agentic/agg-gb200-tp8-agentic.yaml | 141 ++++++++ .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 189 +++++++++++ .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 189 +++++++++++ .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 189 +++++++++++ .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 189 +++++++++++ configs/nvidia-master.yaml | 314 ++++++++++++++++++ perf-changelog.yaml | 12 + runners/launch_gb200-nv.sh | 10 +- 8 files changed, 1229 insertions(+), 4 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml new file mode 100644 index 0000000000..0a27301ad2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -0,0 +1,141 @@ +name: "svf-vllm-agg-gb200-tp8-agentic" + +# AgentX low-concurrency aggregate recipe for DeepSeek-V4-Pro on GB200. One +# TP8 vLLM worker spans two four-GPU nodes and serves both prefill and decode. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + data-parallel-size: 1 + enable-cumem-allocator: true + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + # AgentX session fan-out can exceed the configured live-session + # concurrency; retain scheduler/graph headroom for subagent bursts. + max-num-seqs: 32 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-flashinfer-autotune: true + disable-uvicorn-access-log: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + # One aggregate worker owns all eight GPUs. Use the aggregate accounting + # path instead of summing independent prefill and decode allocations. + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..3568dc9c96 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,189 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep12-agentic" + +# One DEP8 prefill worker feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 3 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml new file mode 100644 index 0000000000..774136735b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -0,0 +1,189 @@ +name: "svf-vllm-disagg-gb200-1p1d-dep8-dep8-agentic" + +# One DEP8 prefill worker feeding one DEP8 decode worker, with attention DP8 / +# expert EP8 and hybrid load balancing on both roles. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 60 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 60 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml new file mode 100644 index 0000000000..d744217dfd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -0,0 +1,189 @@ +name: "svf-vllm-disagg-gb200-2p1d-dep8-dep12-agentic" + +# Two DEP8 prefill workers feeding one DEP12 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 12 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 12 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml new file mode 100644 index 0000000000..d369645818 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -0,0 +1,189 @@ +name: "svf-vllm-disagg-gb200-3p1d-dep8-dep16-agentic" + +# Three DEP8 prefill workers feeding one DEP16 decode worker, with attention / +# expert parallelism matching each role and hybrid load balancing. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +dynamo: + hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb200" + gpus_per_node: 4 + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: "null" + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 120 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.9 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-ep-weight-filter: true + enable-cumem-allocator: true + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4b35c30f87..01be6139c3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,6 +7663,320 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb200-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1, 4, 8, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [64, 128, 256, 320] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [128, 256, 320] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml" + decode: + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [480, 640, 768] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml" + decode: + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [640, 800, 960, 1280] + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + +glm5-fp4-gb300-dynamo-trt-mtp: + image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0-dev.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 15 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [ 84 ] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 180 ] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 333 ] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 10 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 1229 ] + prefill: + num-worker: 14 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 615 ] + prefill: + num-worker: 11 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [ 2253 ] + prefill: + num-worker: 21 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml + - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9e959c9464..e7f11f5002 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4950,3 +4950,15 @@ description: - "Bump vLLM image to v0.25.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 + - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12 + - dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12 + - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 + description: + - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [1,4,8,16] and disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." + - "Aggregate TP8 uses sparse DSV4 attention, NUMA binding, max-num-seqs/CUDA graph size 32, max-num-batched-tokens 32768, and 0.95 GPU memory utilization." + - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/YYYY diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index a5f917da09..baec5b0945 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -293,18 +293,20 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses the same pinned cquil11/srt-slurm-nv commit as - # launch_gb300-nv.sh — everything the agentic recipes need is there: + # DSV4 AgentX DEP recipes require NVIDIA/srt-slurm#90's per-node DP launch + # mode so each process owns the four local DP ranks in one CUDA namespace. + # The pinned commit also provides everything else these recipes need: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) + # - mooncake_kv_store for the external AgentX prefix-cache service # - srtctl apply --no-preflight (model path /mnt/numa1 is compute-node # local NVMe, invisible to the login-node runner) # - benchmark_stage srun_options propagation (container-remap-root # must reach the agentic_srt.sh srun) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout de59739b172e507e15ebf145bfe305f606e82fbf + git checkout 975e1e60d1442f42384b439b80dbaa686498b4f3 mkdir -p recipes/vllm/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic From 156bb261e3eee28ab85f3a9395b0701cc9b81c39 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 09:46:57 +0000 Subject: [PATCH 02/22] update PR number --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e7f11f5002..69d6cf427c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4961,4 +4961,4 @@ - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [1,4,8,16] and disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." - "Aggregate TP8 uses sparse DSV4 attention, NUMA binding, max-num-seqs/CUDA graph size 32, max-num-batched-tokens 32768, and 0.95 GPU memory utilization." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/YYYY + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 From e93a5e28bdf0736b075d6bd725055897d5021b7f Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 09:58:15 +0000 Subject: [PATCH 03/22] chore: update --- runners/launch_gb200-nv.sh | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index baec5b0945..21047d4a39 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -293,23 +293,19 @@ fi # TODO(CJQ): make first class upon srt-slurm upstream refactor if [[ "$IS_AGENTIC" == "1" ]]; then - # DSV4 AgentX DEP recipes require NVIDIA/srt-slurm#90's per-node DP launch - # mode so each process owns the four local DP ranks in one CUDA namespace. - # The pinned commit also provides everything else these recipes need: - # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env - # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (recipes pin the ai-dynamo wheel) - # - mooncake_kv_store for the external AgentX prefix-cache service - # - srtctl apply --no-preflight (model path /mnt/numa1 is compute-node - # local NVMe, invisible to the login-node runner) - # - benchmark_stage srun_options propagation (container-remap-root - # must reach the agentic_srt.sh srun) - git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout 975e1e60d1442f42384b439b80dbaa686498b4f3 - mkdir -p recipes/vllm/deepseek-v4/agentic + # v1.0.27 is the last release with the compatible mooncake_master command; + # v1.0.28 introduced the unsupported --nof_* flag. + # The pinned release also provides the vLLM mooncake_kv_store + SRT_SLURM_AGENTIC_SHA="f6eb42aee4664207dcf2ec601e3bd57bd527efd6" + git clone --branch v1.0.27 --depth 1 https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_AGENTIC_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm v1.0.27 did not resolve to $SRT_SLURM_AGENTIC_SHA" >&2 + exit 1 + fi + mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ - recipes/vllm/deepseek-v4/agentic + recipes/vllm/deepseek-v4/agentic || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" From bafe1b9b450042844cd0bd8940d0ae111cd83200 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 10:21:26 +0000 Subject: [PATCH 04/22] update dynamo --- runners/launch_gb200-nv.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 21047d4a39..3ed1b50828 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -303,6 +303,23 @@ if [[ "$IS_AGENTIC" == "1" ]]; then echo "Error: NVIDIA/srt-slurm v1.0.27 did not resolve to $SRT_SLURM_AGENTIC_SHA" >&2 exit 1 fi + + # ai-dynamo/dynamo#11303. + DYNAMO_SCHEMA="src/srtctl/core/schema.py" + DYNAMO_UPSTREAM_HASH_CLONE=' f"git clone https://github.com/ai-dynamo/dynamo.git && "' + DYNAMO_FORK_HASH_CLONE=' f"git clone https://github.com/esmeetu/dynamo.git && "' + if [[ "$(grep -Fxc "$DYNAMO_UPSTREAM_HASH_CLONE" "$DYNAMO_SCHEMA")" != "1" ]]; then + echo "Error: Could not uniquely locate srt-slurm's hash-pinned Dynamo clone command" >&2 + exit 1 + fi + sed -i \ + 's#f"git clone https://github\.com/ai-dynamo/dynamo\.git && "#f"git clone https://github.com/esmeetu/dynamo.git \&\& "#' \ + "$DYNAMO_SCHEMA" || exit 1 + if [[ "$(grep -Fxc "$DYNAMO_FORK_HASH_CLONE" "$DYNAMO_SCHEMA")" != "1" ]]; then + echo "Error: Failed to redirect the hash-pinned Dynamo clone to esmeetu/dynamo" >&2 + exit 1 + fi + mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic || exit 1 From 60f43a391300e34280994ce26e02ae53a33841cc Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 11:02:20 +0000 Subject: [PATCH 05/22] fix: canonical ckpt for agentic --- runners/launch_gb200-nv.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 3ed1b50828..9457115cd6 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -156,11 +156,15 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then export MODEL_PATH="/mnt/lustre01/models/kimi-k2.5-nvfp4" export SRT_SLURM_MODEL_PREFIX="kimi-k2.5-nvfp4" elif [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp4" ]]; then - # The FP4 checkpoint is staged on compute-visible Lustre. The former - # /mnt/numa1 path is no longer present on watchtower compute nodes; - # the lowercase Lustre sibling is the FP8 checkpoint, so keep the - # NVFP4 path explicit here. - export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + if [[ "${IS_AGENTIC:-0}" == "1" ]]; then + # AgentX was tuned against the canonical DeepSeek-V4-Pro + # checkpoint, which is staged on compute-visible Lustre. + export MODEL_PATH="/mnt/lustre01/models/deepseek-v4-pro" + else + # Existing fixed-sequence GB200 recipes use the NVIDIA ModelOpt + # NVFP4 checkpoint. + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro-NVFP4/" + fi export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/mnt/lustre01/models/MiniMax-M2.5-NVFP4" From 43d433865fd1b594de189113f7d5c9c6fd165f4b Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:33:35 -0400 Subject: [PATCH 06/22] Fix model path casing for DeepSeek-V4-pro --- runners/launch_gb200-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 9457115cd6..2f8bddcc9b 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -159,7 +159,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then if [[ "${IS_AGENTIC:-0}" == "1" ]]; then # AgentX was tuned against the canonical DeepSeek-V4-Pro # checkpoint, which is staged on compute-visible Lustre. - export MODEL_PATH="/mnt/lustre01/models/deepseek-v4-pro" + export MODEL_PATH="/mnt/lustre01/models/deepseek-V4-pro" else # Existing fixed-sequence GB200 recipes use the NVIDIA ModelOpt # NVFP4 checkpoint. From 90ff7e3e25750685191e37dae7b87ebeca2ca26b Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:39:02 -0400 Subject: [PATCH 07/22] Fix model path casing in launch_gb200-nv.sh --- runners/launch_gb200-nv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 2f8bddcc9b..6c0d9bf682 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -159,7 +159,7 @@ elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then if [[ "${IS_AGENTIC:-0}" == "1" ]]; then # AgentX was tuned against the canonical DeepSeek-V4-Pro # checkpoint, which is staged on compute-visible Lustre. - export MODEL_PATH="/mnt/lustre01/models/deepseek-V4-pro" + export MODEL_PATH="/mnt/lustre01/models/DeepSeek-V4-Pro/" else # Existing fixed-sequence GB200 recipes use the NVIDIA ModelOpt # NVFP4 checkpoint. From 3383e63bb45f47852cdc8c7900947f03dd9efa9c Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Fri, 17 Jul 2026 07:36:29 -0700 Subject: [PATCH 08/22] use mooncake RDMA auto-discover --- .../deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 11 ++++------- .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 14 +++++++------- .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 14 +++++++------- .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 14 +++++++------- .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 14 +++++++------- 5 files changed, 32 insertions(+), 35 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 0a27301ad2..21433bf3ca 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -53,7 +53,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + device_name: "" mode: "embedded" enable_offload: false aggregated_environment: @@ -84,9 +84,9 @@ backend: UCX_TLS: "cuda_copy,cuda_ipc,tcp" NCCL_P2P_LEVEL: NVL MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" vllm_config: aggregated: @@ -131,9 +131,6 @@ benchmark: # One aggregate worker owns all eight GPUs. Use the aggregate accounting # path instead of summing independent prefill and decode allocations. IS_MULTINODE: "false" - TP: "8" - EP_SIZE: "1" - DP_ATTENTION: "false" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 3568dc9c96..0a40dd29c9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -59,7 +59,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + device_name: "" mode: "embedded" enable_offload: false prefill_environment: @@ -87,9 +87,9 @@ backend: VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" decode_environment: DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" @@ -112,9 +112,9 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" vllm_config: prefill: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index 774136735b..43d5fd9de6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -59,7 +59,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + device_name: "" mode: "embedded" enable_offload: false prefill_environment: @@ -87,9 +87,9 @@ backend: VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" decode_environment: DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" @@ -112,9 +112,9 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" vllm_config: prefill: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index d744217dfd..f6686bcf04 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -59,7 +59,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + device_name: "" mode: "embedded" enable_offload: false prefill_environment: @@ -87,9 +87,9 @@ backend: VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" decode_environment: DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" @@ -112,9 +112,9 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" vllm_config: prefill: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index d369645818..f07068f3cb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -59,7 +59,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + device_name: "" mode: "embedded" enable_offload: false prefill_environment: @@ -87,9 +87,9 @@ backend: VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" decode_environment: DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" @@ -112,9 +112,9 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" + # MC_STORE_CLIENT_METRIC: "1" + # MC_STORE_CLIENT_METRIC_INTERVAL: "5" + # MC_TE_METRIC: "0" vllm_config: prefill: From af6aec6052f9c9b7830c01c5e29c5ae9a41b1edf Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Fri, 17 Jul 2026 07:54:19 -0700 Subject: [PATCH 09/22] fix --- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 21433bf3ca..ff38364969 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -130,7 +130,12 @@ benchmark: PORT: "8000" # One aggregate worker owns all eight GPUs. Use the aggregate accounting # path instead of summing independent prefill and decode allocations. + # Aggregate (IS_MULTINODE=false) uses the single-node topology labels below + # directly (num_gpus = TP), so they are set here rather than CI-injected. IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" From e366444b26e95ac4b4698f511e2e5427bd66067c Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Fri, 17 Jul 2026 08:02:21 -0700 Subject: [PATCH 10/22] remove disable-uvicorn-access-log --- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index ff38364969..2027d1039b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -106,7 +106,6 @@ backend: max-num-batched-tokens: 32768 trust-remote-code: true no-enable-flashinfer-autotune: true - disable-uvicorn-access-log: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 32 From c1a2917411007597cad942bd4e0be709cfbf29b5 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Fri, 17 Jul 2026 09:47:06 -0700 Subject: [PATCH 11/22] Gate dynamo conv aware routing with env var and set to 0 for now due to dynamo version mismatch vs aiperf --- benchmarks/benchmark_lib.sh | 6 +++++- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 3 ++- .../agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 3 ++- .../agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 3 ++- .../agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 3 ++- .../agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 3 ++- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 2ba3d8923d..5fd8b94601 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 2027d1039b..2e25d75d73 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -135,7 +135,8 @@ benchmark: TP: "8" EP_SIZE: "1" DP_ATTENTION: "false" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 0a40dd29c9..95602c854f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -182,7 +182,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index 43d5fd9de6..d4822f99e6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -182,7 +182,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index f6686bcf04..16ed534536 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -182,7 +182,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index f07068f3cb..48e66a406a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -182,7 +182,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Off: aiperf emits nvext.session_control, rejected by current dynamo (see benchmark_lib.sh). + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" From 453e3423dfe77e3d436b1cc6a7d4e83102a8ce09 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 03:36:09 +0000 Subject: [PATCH 12/22] fix: (1) tp config; (2) fresh start per conc --- .../agentic/agg-gb200-tp8-agentic.yaml | 1 - utils/matrix_logic/generate_sweep_configs.py | 4 ++- .../test_generate_sweep_configs.py | 31 +++++++++++++------ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 2e25d75d73..b88499dadf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -95,7 +95,6 @@ backend: kv-cache-dtype: "fp8" tensor-parallel-size: 8 pipeline-parallel-size: 1 - data-parallel-size: 1 enable-cumem-allocator: true numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e14c50c66b..5272fe78e6 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,9 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# One task/Slurm allocation per concurrency gives each benchmark point a +# fresh server deployment, matching single-node agentic sweep isolation. +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 9a1c67d86e..8d23b83a18 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2253,8 +2253,8 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf with pytest.raises(ValueError, match="exceeds gpus-per-node"): generate_test_config_sweep(args, config, runner_config) - def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + def test_multinode_agentic_uses_one_allocation_per_concurrency(self): + """Each concurrency should get its own server allocation.""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2292,17 +2292,27 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x4_d1x4_conc16x32x64x128" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [ + [16], + [32], + [64], + [128], + [256], + ] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x4_d1x4_conc16", + "dsv4_p2x4_d1x4_conc32", + "dsv4_p2x4_d1x4_conc64", + "dsv4_p2x4_d1x4_conc128", + "dsv4_p2x4_d1x4_conc256", + ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 assert result[0]["prefill"]["pcp-size"] == 2 assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x4_d1x4_conc256" assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) assert all(entry["kv-p2p-transfer"] == "nixl" for entry in result) @@ -2493,9 +2503,10 @@ def test_node_type_filters_apply_to_agentic_configs( assert "prefill" not in single_result[0] assert single_result[0]["runner"] == "cluster:b300-nv" assert single_result[0]["pp"] == 2 - assert len(multi_result) == 1 - assert "prefill" in multi_result[0] - assert multi_result[0]["runner"] == "cluster:gb200-nv" + assert len(multi_result) == 2 + assert [entry["conc"] for entry in multi_result] == [[16], [32]] + assert all("prefill" in entry for entry in multi_result) + assert all(entry["runner"] == "cluster:gb200-nv" for entry in multi_result) assert ( multi_result[0]["prefill"]["pp"], multi_result[0]["prefill"]["dcp-size"], From 5d7c2b065ced163149a8225a0476264c7816d9e6 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 04:44:26 +0000 Subject: [PATCH 13/22] feat: enable per_node --- .../agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 1 + .../agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 1 + .../agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 1 + .../agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 1 + runners/launch_gb200-nv.sh | 11 +++++++++++ 5 files changed, 15 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 95602c854f..2f96a36060 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -50,6 +50,7 @@ frontend: backend: type: vllm + dp_launch_mode: per_node connector: "null" kv_events_config: prefill: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index d4822f99e6..38fdf96ccc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -50,6 +50,7 @@ frontend: backend: type: vllm + dp_launch_mode: per_node connector: "null" kv_events_config: prefill: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index 16ed534536..c639b33f86 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -50,6 +50,7 @@ frontend: backend: type: vllm + dp_launch_mode: per_node connector: "null" kv_events_config: prefill: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index 48e66a406a..1fd705e2bb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -50,6 +50,7 @@ frontend: backend: type: vllm + dp_launch_mode: per_node connector: "null" kv_events_config: prefill: true diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 6c0d9bf682..1ef050bb35 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -308,6 +308,17 @@ if [[ "$IS_AGENTIC" == "1" ]]; then exit 1 fi + # Backport NVIDIA/srt-slurm#90 without taking the post-v1.0.27 + # mooncake_master changes. The feature launches one vLLM process per node + # and lets that process manage all node-local data-parallel ranks. + SRT_SLURM_PER_NODE_DP_SHA="1a0f9e3633318ab1ee9428d2129161b583786b18" + git fetch --depth 2 origin refs/pull/90/head || exit 1 + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_PER_NODE_DP_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm PR #90 commit did not resolve to $SRT_SLURM_PER_NODE_DP_SHA" >&2 + exit 1 + fi + git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" || exit 1 + # ai-dynamo/dynamo#11303. DYNAMO_SCHEMA="src/srtctl/core/schema.py" DYNAMO_UPSTREAM_HASH_CLONE=' f"git clone https://github.com/ai-dynamo/dynamo.git && "' From a94e310b3b6a81547591e9a31a0e82f2d8a69b9c Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 04:44:53 +0000 Subject: [PATCH 14/22] chore: adjust conc list --- .../agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 6 +++--- .../agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 2 +- .../agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 6 +++--- configs/nvidia-master.yaml | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 2f96a36060..f4b8aaf4a6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -134,7 +134,7 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 120 + max-num-seqs: 256 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index 38fdf96ccc..61bc7d488f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -134,7 +134,7 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 120 + max-num-seqs: 128 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true @@ -158,13 +158,13 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 60 + max-num-seqs: 64 max-num-batched-tokens: 256 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 60 + max-cudagraph-capture-size: 64 gpu-memory-utilization: 0.95 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index c639b33f86..e7527b5774 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -134,7 +134,7 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 120 + max-num-seqs: 192 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index 1fd705e2bb..eb472b4571 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -134,7 +134,7 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 120 + max-num-seqs: 214 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true @@ -158,13 +158,13 @@ backend: numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 - max-num-seqs: 128 + max-num-seqs: 160 max-num-batched-tokens: 256 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 + max-cudagraph-capture-size: 160 gpu-memory-utilization: 0.95 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 01be6139c3..0c43b21055 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7708,7 +7708,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: agentic-coding: - search-space: - spec-decoding: none - conc-list: [64, 128, 256, 320] + conc-list: [64, 128, 192, 256] prefill: num-worker: 1 tp: 8 @@ -7737,7 +7737,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12: agentic-coding: - search-space: - spec-decoding: none - conc-list: [128, 256, 320] + conc-list: [384, 512] prefill: num-worker: 1 tp: 8 @@ -7766,7 +7766,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12: agentic-coding: - search-space: - spec-decoding: none - conc-list: [480, 640, 768] + conc-list: [640, 720, 768] prefill: num-worker: 2 tp: 8 @@ -7795,7 +7795,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: agentic-coding: - search-space: - spec-decoding: none - conc-list: [640, 800, 960, 1280] + conc-list: [800, 960, 1024, 1280] prefill: num-worker: 3 tp: 8 From 6f86c85f26da585733cbf76cb2f37854c66ca19e Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Sat, 18 Jul 2026 17:19:23 -0700 Subject: [PATCH 15/22] fixup --- .../agentic/agg-gb200-tp8-agentic.yaml | 4 +- .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 4 +- .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 4 +- .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 4 +- .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 4 +- configs/nvidia-master.yaml | 64 ++++++++++--------- perf-changelog.yaml | 4 +- 7 files changed, 51 insertions(+), 37 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index b88499dadf..8a96f6f3bc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -53,7 +53,9 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false aggregated_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index f4b8aaf4a6..fda6cb550f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -60,7 +60,9 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false prefill_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index 61bc7d488f..cea0ad3047 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -60,7 +60,9 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false prefill_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index e7527b5774..07f930fa6a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -60,7 +60,9 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false prefill_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index eb472b4571..1bb6d455c6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -60,7 +60,9 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + # Pin to the IB compute fabric. device_name: "" auto-discovers RoCE/link-local + # NICs (mlx5_2/mlx5_5) that can not route cross-node and hang mooncake setup. + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" mode: "embedded" enable_offload: false prefill_environment: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0c43b21055..3dbb6b2f28 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,35 +7663,41 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true -dsv4-fp4-gb200-dynamo-vllm-agentic-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - multinode: true - disagg: false - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [1, 4, 8, 16] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false +# TEMPORARILY DISABLED: the TP8 aggregate is a single vLLM engine with TP=8 +# spanning 2 GB200 nodes (4 GPUs each). It hits a vLLM shm-broadcast port race +# on the follower node -- the 4 remote-node TP workers all call get_open_port() +# from the same VLLM_PORT and collide (EADDRINUSE) with no retry, crashing the +# engine. Re-enable once that vLLM race is fixed. The disagg configs below use +# per-node DEP (not cross-node single-engine TP8) and are unaffected. +# dsv4-fp4-gb200-dynamo-vllm-agentic-agg: +# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 +# model: deepseek-ai/DeepSeek-V4-Pro +# model-prefix: dsv4 +# runner: cluster:gb200-nv +# precision: fp4 +# framework: dynamo-vllm +# router: { name: dynamo-router, version: "1.3.0.dev20260618" } +# multinode: true +# disagg: false +# scenarios: +# agentic-coding: +# - search-space: +# - spec-decoding: none +# conc-list: [1, 4, 8, 16] +# prefill: +# num-worker: 1 +# tp: 8 +# ep: 1 +# dp-attn: false +# additional-settings: +# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" +# # The aggregate worker also performs decode; keep the decode worker +# # count at zero so result aggregation counts eight GPUs only once. +# decode: +# num-worker: 0 +# tp: 8 +# ep: 1 +# dp-attn: false dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 69d6cf427c..684d4dbc19 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4952,13 +4952,11 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 - config-keys: - - dsv4-fp4-gb200-dynamo-vllm-agentic-agg - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12 - dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12 - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [1,4,8,16] and disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." - - "Aggregate TP8 uses sparse DSV4 attention, NUMA binding, max-num-seqs/CUDA graph size 32, max-num-batched-tokens 32768, and 0.95 GPU memory utilization." + - "Add GB200 Dynamo-vLLM AgentX disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 From 7778d8b8ad67b15f3a6f8629a689f32122b18e10 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Sat, 18 Jul 2026 17:38:40 -0700 Subject: [PATCH 16/22] Use dynamo main instead of custom branch --- .../agentic/agg-gb200-tp8-agentic.yaml | 2 +- .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 2 +- .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 2 +- .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 2 +- .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 2 +- runners/launch_gb200-nv.sh | 18 +++--------------- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 8a96f6f3bc..39fda80ed6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -9,7 +9,7 @@ model: precision: "fp4" dynamo: - hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index fda6cb550f..dc48a85e25 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -9,7 +9,7 @@ model: precision: "fp4" dynamo: - hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index cea0ad3047..98b00146f5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -9,7 +9,7 @@ model: precision: "fp4" dynamo: - hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index 07f930fa6a..53997da609 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -9,7 +9,7 @@ model: precision: "fp4" dynamo: - hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index 1bb6d455c6..49a8879eea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -9,7 +9,7 @@ model: precision: "fp4" dynamo: - hash: "6972dd2a39aa61c5dbfdf7e901ca859111897643" + hash: "1f74ef8c204ed6e283c22374f2b8c5cc83bf7b52" install: true setup_script: vllm-container-deps.sh diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 1ef050bb35..d7c3eafe1e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -319,21 +319,9 @@ if [[ "$IS_AGENTIC" == "1" ]]; then fi git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" || exit 1 - # ai-dynamo/dynamo#11303. - DYNAMO_SCHEMA="src/srtctl/core/schema.py" - DYNAMO_UPSTREAM_HASH_CLONE=' f"git clone https://github.com/ai-dynamo/dynamo.git && "' - DYNAMO_FORK_HASH_CLONE=' f"git clone https://github.com/esmeetu/dynamo.git && "' - if [[ "$(grep -Fxc "$DYNAMO_UPSTREAM_HASH_CLONE" "$DYNAMO_SCHEMA")" != "1" ]]; then - echo "Error: Could not uniquely locate srt-slurm's hash-pinned Dynamo clone command" >&2 - exit 1 - fi - sed -i \ - 's#f"git clone https://github\.com/ai-dynamo/dynamo\.git && "#f"git clone https://github.com/esmeetu/dynamo.git \&\& "#' \ - "$DYNAMO_SCHEMA" || exit 1 - if [[ "$(grep -Fxc "$DYNAMO_FORK_HASH_CLONE" "$DYNAMO_SCHEMA")" != "1" ]]; then - echo "Error: Failed to redirect the hash-pinned Dynamo clone to esmeetu/dynamo" >&2 - exit 1 - fi + # ai-dynamo/dynamo#11303 is merged into ai-dynamo/dynamo main, so the + # recipe-pinned dynamo hash resolves against upstream directly -- no + # esmeetu/dynamo fork redirect of srt-slurm's schema.py needed anymore. mkdir -p recipes/vllm/deepseek-v4/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ From a43af120dac43adb3e8e8a4a2c19734c84999bc9 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Sat, 18 Jul 2026 18:06:56 -0700 Subject: [PATCH 17/22] Remove router-reset-states: true --- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 1 - .../agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 1 - .../deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 1 - .../agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 1 - .../deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml | 1 - .../agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 1 - .../deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml | 1 - 7 files changed, 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 39fda80ed6..16a5f66d02 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -37,7 +37,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-temperature: 0.0 router-queue-threshold: 65536 active-decode-blocks-threshold: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index dc48a85e25..24f4c1e359 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -40,7 +40,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-temperature: 0.0 router-queue-threshold: 65536 active-decode-blocks-threshold: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index 98b00146f5..b2c7647840 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -40,7 +40,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-temperature: 0.0 router-queue-threshold: 65536 active-decode-blocks-threshold: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index 53997da609..d0ff21f83d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -40,7 +40,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-temperature: 0.0 router-queue-threshold: 65536 active-decode-blocks-threshold: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml index b647276179..f3539b4a9e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml @@ -47,7 +47,6 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" - router-reset-states: true backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index 49a8879eea..b512cc8699 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -40,7 +40,6 @@ frontend: enable_multiple_frontends: false args: router-mode: "kv" - router-reset-states: true router-temperature: 0.0 router-queue-threshold: 65536 active-decode-blocks-threshold: "None" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml index 9e2b665843..4eae7c103e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml @@ -46,7 +46,6 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" - router-reset-states: true backend: type: vllm From 40da599f97ba132ae30faabb5b06eb9bf6f488c7 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Sat, 18 Jul 2026 19:54:01 -0700 Subject: [PATCH 18/22] Re-add tp8 config with patch --- .../agentic/agg-gb200-tp8-agentic.yaml | 14 +++- .../disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 2 +- .../disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 2 +- .../disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 2 +- .../disagg-gb200-2p1d-dep8-dep8-agentic.yaml | 2 +- .../disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 2 +- .../disagg-gb200-3p2d-tep8-tp8-agentic.yaml | 2 +- configs/nvidia-master.yaml | 64 +++++++++---------- perf-changelog.yaml | 3 +- runners/launch_gb200-nv.sh | 8 +++ .../srt-slurm-vllm-port-single-gpu.patch | 36 +++++++++++ 11 files changed, 92 insertions(+), 45 deletions(-) create mode 100644 runners/patches/srt-slurm-vllm-port-single-gpu.patch diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 16a5f66d02..f12d652a09 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -18,7 +18,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 2160 + max_attempts: 720 interval_seconds: 10 resources: @@ -63,7 +63,11 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" + # Off: the c188b96 container's CUDA-graph capture path enters vLLM's + # nccl_symm_mem_context without set_graph_pool_id, so capture asserts + # "graph_pool_id is not set under graph capture". Re-enable once this recipe + # moves to a container built from a vLLM with the cuda_graph.py fix. + VLLM_USE_NCCL_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" @@ -97,7 +101,11 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 1 enable-cumem-allocator: true - numa-bind: true + # numa-bind disabled: srt-slurm's cpus-per-task/cgroup already constrains + # CPU affinity, so vLLM's auto-NUMA detection bails and --numa-bind hard- + # errors ("could not detect the GPU-to-NUMA topology"). Re-enable only with + # an explicit --numa-bind-nodes mapping for this cluster. + # numa-bind: true attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' max-model-len: 1048576 # AgentX session fan-out can exceed the configured live-session diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 24f4c1e359..8d09942af6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -18,7 +18,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 2160 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index b2c7647840..e367b81949 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -18,7 +18,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 2160 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index d0ff21f83d..4fb9fa7e2a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -18,7 +18,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 2160 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml index f3539b4a9e..3504cb99c8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml @@ -20,7 +20,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 1440 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index b512cc8699..503f3d3ea3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -18,7 +18,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 2160 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml index 4eae7c103e..5be157f273 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml @@ -21,7 +21,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 1440 + max_attempts: 720 interval_seconds: 10 resources: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3dbb6b2f28..0c43b21055 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,41 +7663,35 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true -# TEMPORARILY DISABLED: the TP8 aggregate is a single vLLM engine with TP=8 -# spanning 2 GB200 nodes (4 GPUs each). It hits a vLLM shm-broadcast port race -# on the follower node -- the 4 remote-node TP workers all call get_open_port() -# from the same VLLM_PORT and collide (EADDRINUSE) with no retry, crashing the -# engine. Re-enable once that vLLM race is fixed. The disagg configs below use -# per-node DEP (not cross-node single-engine TP8) and are unaffected. -# dsv4-fp4-gb200-dynamo-vllm-agentic-agg: -# image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 -# model: deepseek-ai/DeepSeek-V4-Pro -# model-prefix: dsv4 -# runner: cluster:gb200-nv -# precision: fp4 -# framework: dynamo-vllm -# router: { name: dynamo-router, version: "1.3.0.dev20260618" } -# multinode: true -# disagg: false -# scenarios: -# agentic-coding: -# - search-space: -# - spec-decoding: none -# conc-list: [1, 4, 8, 16] -# prefill: -# num-worker: 1 -# tp: 8 -# ep: 1 -# dp-attn: false -# additional-settings: -# - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" -# # The aggregate worker also performs decode; keep the decode worker -# # count at zero so result aggregation counts eight GPUs only once. -# decode: -# num-worker: 0 -# tp: 8 -# ep: 1 -# dp-attn: false +dsv4-fp4-gb200-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.0.dev20260618" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1, 4, 8, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 684d4dbc19..bab0b22b4e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4952,11 +4952,12 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 - config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-agg - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep8 - dsv4-fp4-gb200-dynamo-vllm-agentic-1p1d-dep8-dep12 - dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep12 - dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - - "Add GB200 Dynamo-vLLM AgentX disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." + - "Add GB200 Dynamo-vLLM AgentX aggregate TP8 at conc [1,4,8,16] and disaggregated topologies: 1P/1D DEP8/DEP8 at [64,128,256,320], 1P/1D DEP8/DEP12 at [128,256,320], 2P/1D DEP8/DEP12 at [480,640,768], and 3P/1D DEP8/DEP16 at [640,800,960,1280]." - "Image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index d7c3eafe1e..cfcf71d6e5 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -319,6 +319,14 @@ if [[ "$IS_AGENTIC" == "1" ]]; then fi git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" || exit 1 + # Pin VLLM_PORT only for single-GPU processes. Multi-GPU worker processes + # (multi-node tensor parallel, e.g. the TP8 aggregate spanning two 4-GPU + # nodes) run vLLM's internal multiproc executor whose same-node subprocesses + # otherwise all read one VLLM_PORT and race to bind the shm-broadcast port + # (EADDRINUSE crash). Not yet in an srt-slurm release, so apply it here as a + # patch until it lands upstream. + git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-vllm-port-single-gpu.patch" || exit 1 + # ai-dynamo/dynamo#11303 is merged into ai-dynamo/dynamo main, so the # recipe-pinned dynamo hash resolves against upstream directly -- no # esmeetu/dynamo fork redirect of srt-slurm's schema.py needed anymore. diff --git a/runners/patches/srt-slurm-vllm-port-single-gpu.patch b/runners/patches/srt-slurm-vllm-port-single-gpu.patch new file mode 100644 index 0000000000..39a879c14a --- /dev/null +++ b/runners/patches/srt-slurm-vllm-port-single-gpu.patch @@ -0,0 +1,36 @@ +diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py +index 0484d70..f3a95ca 100644 +--- a/src/srtctl/backends/vllm.py ++++ b/src/srtctl/backends/vllm.py +@@ -284,7 +284,8 @@ class VLLMProtocol: + multi-node NIXL handshake) + - VLLM_PORT: private base for vLLM's get_open_port() scans, unique per + process so co-located workers don't race for the same rendezvous port +- (see the notes on VLLM_PORT_BASE in srtctl.ports) ++ (see the notes on VLLM_PORT_BASE in srtctl.ports). Set ONLY for ++ single-GPU processes -- see below. + """ + from srtctl.core.slurm import get_hostname_ip + +@@ -299,8 +300,19 @@ class VLLMProtocol: + # 4xGB200 nodes: each prefill endpoint is DEP2 (uses 2 of the 4 GPUs), so + # two endpoints share one physical node and would otherwise scan + # overlapping get_open_port() ranges. +- proc_index = max(process.sys_port - DYN_SYSTEM_PORT_BASE, 0) +- env["VLLM_PORT"] = str(VLLM_PORT_BASE + proc_index * VLLM_PORT_STRIDE) ++ # ++ # Only pin it for single-GPU processes (the per-GPU DP layout). A ++ # multi-GPU process runs vLLM's internal multiproc executor, whose N ++ # same-node worker subprocesses all read this one VLLM_PORT; in a ++ # multi-node TP group the remote-node subprocesses then race to bind the ++ # shm-broadcast port from that shared base and crash with EADDRINUSE ++ # (e.g. the TP8 aggregate spanning two 4-GPU nodes). Leaving VLLM_PORT ++ # unset lets those subprocesses fall back to OS-assigned ephemeral ports, ++ # which are unique per bind. Single-node multi-GPU TP never hit this ++ # (same-node reader -> IPC, no TCP bind), so this is safe there too. ++ if len(process.gpu_indices) == 1: ++ proc_index = max(process.sys_port - DYN_SYSTEM_PORT_BASE, 0) ++ env["VLLM_PORT"] = str(VLLM_PORT_BASE + proc_index * VLLM_PORT_STRIDE) + return env + + def get_mooncake_worker_env(self, infra_node_ip: str, local_hostname: str) -> dict[str, str]: From f0c052a45862fb78fec09c574addf426fa40f8e7 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 08:19:11 +0000 Subject: [PATCH 19/22] fix(gb200): correct per-node Dynamo health counts Backport srt-slurm health expectations for per-node vLLM data-parallel launches and apply the patch from the GB200 agentic launcher. --- .../srt-slurm-vllm-per-node-health.patch | 63 +++++++++++++++++++ runners/launch_gb200-nv.sh | 7 +++ 2 files changed, 70 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch diff --git a/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch new file mode 100644 index 0000000000..d42553fb61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch @@ -0,0 +1,63 @@ +diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py +index fe6cbf1..f282fe6 100644 +--- a/src/srtctl/cli/mixins/benchmark_stage.py ++++ b/src/srtctl/cli/mixins/benchmark_stage.py +@@ -46,12 +46,32 @@ def _vllm_data_parallel_size(config: "SrtConfig", mode: str) -> int: + return int(mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size") or 1) + + +-def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: ++def _vllm_health_entries( ++ config: "SrtConfig", ++ mode: str, ++ logical_workers: int, ++ backend_processes: list["Process"] | None, ++) -> int: ++ """Return expected Dynamo generate registrations for a vLLM worker mode.""" ++ dp_size = _vllm_data_parallel_size(config, mode) ++ if dp_size > 1 and getattr(config.backend, "dp_launch_mode", "per_gpu") == "per_node": ++ if backend_processes is None: ++ raise ValueError("backend_processes are required for per-node DP health expectations") ++ endpoint_mode = "agg" if mode == "aggregated" else mode ++ return sum(process.endpoint_mode == endpoint_mode for process in backend_processes) ++ ++ return logical_workers * dp_size ++ ++ ++def _get_health_expectations( ++ config: "SrtConfig", backend_processes: list["Process"] | None = None ++) -> tuple[int, int, str, int]: + """Compute expected health counts in the units reported by the frontend. + + Dynamo's /health endpoint reports registered generate instances. For vLLM +- DP workers, that means one entry per DP rank, not one entry per logical +- srt-slurm worker. Other frontends keep using logical worker counts. ++ DP workers, per-GPU launch registers one entry per DP rank, while per-node ++ launch registers one entry per node-local process. Other frontends keep ++ using logical worker counts. + """ + r = config.resources + +@@ -67,10 +87,10 @@ def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: + if config.frontend.type == "dynamo" and getattr(config.backend, "type", None) == "vllm": + if r.num_agg > 0: + n_prefill = 0 +- n_decode = logical_decode * _vllm_data_parallel_size(config, "aggregated") ++ n_decode = _vllm_health_entries(config, "aggregated", logical_decode, backend_processes) + else: +- n_prefill = logical_prefill * _vllm_data_parallel_size(config, "prefill") +- n_decode = logical_decode * _vllm_data_parallel_size(config, "decode") ++ n_prefill = _vllm_health_entries(config, "prefill", logical_prefill, backend_processes) ++ n_decode = _vllm_health_entries(config, "decode", logical_decode, backend_processes) + + count_desc = f"{n_prefill}P + {n_decode}D Dynamo generate instances; logical workers: {worker_desc}" + return n_prefill, n_decode, count_desc, n_prefill + n_decode +@@ -131,7 +151,7 @@ class BenchmarkStageMixin: + """Run the benchmark.""" + logger.info("Waiting for workers to be ready...") + +- n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config) ++ n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config, self.backend_processes) + logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) + + hc = self.config.health_check diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index cfcf71d6e5..29fea2a340 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -327,6 +327,13 @@ if [[ "$IS_AGENTIC" == "1" ]]; then # patch until it lands upstream. git apply "$GITHUB_WORKSPACE/runners/patches/srt-slurm-vllm-port-single-gpu.patch" || exit 1 + # Per-node DP launches one Dynamo generate endpoint per node-local process, + # not one per DP rank. Backport the health-count fix from + # ivanium/srt-slurm@ca0880138fa606130ae4acbb8d0afddfb84c69fa. + SRT_SLURM_PER_NODE_HEALTH_PATCH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch" + git apply --check "$SRT_SLURM_PER_NODE_HEALTH_PATCH" || exit 1 + git apply "$SRT_SLURM_PER_NODE_HEALTH_PATCH" || exit 1 + # ai-dynamo/dynamo#11303 is merged into ai-dynamo/dynamo main, so the # recipe-pinned dynamo hash resolves against upstream directly -- no # esmeetu/dynamo fork redirect of srt-slurm's schema.py needed anymore. From 94bb99c5be2892d738dbc71e6038052756cae39d Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Sun, 19 Jul 2026 16:38:52 -0700 Subject: [PATCH 20/22] fix tp8 agg memory --- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index f12d652a09..8912ea8bf0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -111,13 +111,14 @@ backend: # AgentX session fan-out can exceed the configured live-session # concurrency; retain scheduler/graph headroom for subagent bursts. max-num-seqs: 32 - max-num-batched-tokens: 32768 + max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 32 gpu-memory-utilization: 0.95 + kv-cache-memory: 37580963840 # 35 GiB stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 From 9bb72506b463dc1e706e786dcf0b6fa814173082 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Mon, 20 Jul 2026 10:04:06 -0700 Subject: [PATCH 21/22] fix(dsv4-agentic): raise ETCD_LEASE_TTL to 600s on GB200 AgentX recipes Dynamo's primary etcd lease defaults to a 10s TTL. During the ~380s cold start (weight load + cudagraph capture), a brief etcd-server-side stall (etcd is co-located with a worker; peak node memory/IO pressure) revokes every worker's lease at once. Registration then fails with "requested lease not found", workers never go ready, and the health check waits the full 2h before timing out. Observed on the 3p1d-dep8-dep16 run: all 10 workers across 10 nodes, leases granted at staggered times, expired simultaneously within 30ms. A byte-identical rerun passed because no qualifying etcd stall landed in the window -- confirming it's a TTL-gated race, not a fixed config bug. 600s rides through transient etcd stalls; the only cost, slower cleanup of a genuinely dead worker, is handled independently by the health check and the process fail-fast monitor. Applies to the 5 configs added in PR #2260 (agg + 1p1d-dep8-dep8 + 1p1d-dep8-dep12 + 2p1d-dep8-dep12 + 3p1d-dep8-dep16). Co-Authored-By: Claude Opus 4.8 --- .../deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml | 10 ++++++++++ .../agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml | 10 ++++++++++ .../agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml | 10 ++++++++++ .../agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml | 11 +++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml index 8912ea8bf0..9af98f8611 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-agentic.yaml @@ -59,7 +59,7 @@ backend: enable_offload: false aggregated_environment: DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" + ETCD_LEASE_TTL: "600" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml index 8d09942af6..e00d40cee9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep12-agentic.yaml @@ -65,6 +65,13 @@ backend: mode: "embedded" enable_offload: false prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" @@ -93,6 +100,9 @@ backend: # MC_STORE_CLIENT_METRIC_INTERVAL: "5" # MC_TE_METRIC: "0" decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml index e367b81949..400eb7dbe6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-agentic.yaml @@ -65,6 +65,13 @@ backend: mode: "embedded" enable_offload: false prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" @@ -93,6 +100,9 @@ backend: # MC_STORE_CLIENT_METRIC_INTERVAL: "5" # MC_TE_METRIC: "0" decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml index 4fb9fa7e2a..a5d8a6aaff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep12-agentic.yaml @@ -65,6 +65,13 @@ backend: mode: "embedded" enable_offload: false prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. A brief etcd stall + # during the CPU/memory/IO-heavy cold start (~380s weight load, etcd + # co-located with a worker) then revokes every worker's lease at once, so + # registration fails with "requested lease not found" and workers never go + # ready (health check times out at 2h). 600s rides through transient etcd + # stalls; longer TTL only delays dead-worker cleanup, handled independently. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" @@ -93,6 +100,9 @@ backend: # MC_STORE_CLIENT_METRIC_INTERVAL: "5" # MC_TE_METRIC: "0" decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml index 503f3d3ea3..663c2cf09b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p1d-dep8-dep16-agentic.yaml @@ -65,6 +65,14 @@ backend: mode: "embedded" enable_offload: false prefill_environment: + # dynamo's primary etcd lease defaults to a 10s TTL. During the ~380s + # weight load the process is CPU-saturated and the Rust keep-alive task + # cannot renew in time, so etcd revokes the lease and worker registration + # later fails with "requested lease not found" (workers never go ready -> + # health check times out at 2h). 600s covers the weight load + cudagraph + # stalls; longer TTL only delays dead-worker cleanup, which the health + # check / fail-fast monitor handle independently. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" @@ -93,6 +101,9 @@ backend: # MC_STORE_CLIENT_METRIC_INTERVAL: "5" # MC_TE_METRIC: "0" decode_environment: + # See prefill_environment: raise dynamo's primary etcd lease TTL from the + # 10s default so the lease survives the CPU-saturated cold start. + ETCD_LEASE_TTL: "600" DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx" TORCH_SYMMMEM: "NVSHMEM" VLLM_USE_NCCL_SYMM_MEM: "1" From a7b253a092c0be098e1e8f2c15ac3329baaedc36 Mon Sep 17 00:00:00 2001 From: "Wei Zhao (Engrg-Hardware 1)" Date: Mon, 20 Jul 2026 10:18:46 -0700 Subject: [PATCH 22/22] chore(agentx): drop stray glm5 config, revert out-of-scope recipe edits - Remove glm5-fp4-gb300-dynamo-trt-mtp from configs/nvidia-master.yaml: it was accidentally re-added by 6b06ade77 to the active master, but it belongs to PR #1799 (GLM-5 GB300 TRT-LLM) and lives in configs/deprecated on main -- unrelated to this DSV4 GB200 AgentX PR (#2260). Restores main's state; the historical #1799 changelog entry is left intact. - Revert disagg-gb200-2p1d-dep8-dep8 and disagg-gb200-3p2d-tep8-tp8 to main (they are not among the 5 PR #2260 config-keys; the max_attempts and router-reset-states edits were out of scope). nvidia-master.yaml now adds only the 5 dsv4-fp4-gb200-dynamo-vllm-agentic-* config-keys this PR owns. Co-Authored-By: Claude Opus 4.8 --- .../disagg-gb200-2p1d-dep8-dep8-agentic.yaml | 3 +- .../disagg-gb200-3p2d-tep8-tp8-agentic.yaml | 3 +- configs/nvidia-master.yaml | 168 ------------------ 3 files changed, 4 insertions(+), 170 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml index 3504cb99c8..b647276179 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml @@ -20,7 +20,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 720 + max_attempts: 1440 interval_seconds: 10 resources: @@ -47,6 +47,7 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" + router-reset-states: true backend: type: vllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml index 5be157f273..9e2b665843 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml @@ -21,7 +21,7 @@ slurm: time_limit: "8:00:00" health_check: - max_attempts: 720 + max_attempts: 1440 interval_seconds: 10 resources: @@ -46,6 +46,7 @@ frontend: ETCD_LEASE_TTL: "120" args: router-mode: "kv" + router-reset-states: true backend: type: vllm diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c40806c56c..44bf70ce61 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7809,174 +7809,6 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-3p1d-dep8-dep16: ep: 16 dp-attn: true -glm5-fp4-gb300-dynamo-trt-mtp: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "v1.3.0-dev.1" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 15 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx1dep2_gen5tep4_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 84 ] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx2dep2_gen1dep32_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx4dep2_gen1dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 333 ] - prefill: - num-worker: 6 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx6dep2_gen1dep32_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 615 ] - prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx10dep2_gen1dep16_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 1229 ] - prefill: - num-worker: 14 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx14dep2_gen1dep16_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 615 ] - prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx11dep2_gen1dep32_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 2253 ] - prefill: - num-worker: 21 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/MTP/ctx21dep2_gen1dep16_batch128_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro