From d02cb0d98cda139c6707b543a7341c563c03aaaf Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Fri, 17 Jul 2026 08:24:45 +0000 Subject: [PATCH 01/14] chore: port basic settings from gb200 disagg --- benchmarks/benchmark_lib.sh | 6 +++- utils/matrix_logic/generate_sweep_configs.py | 4 ++- .../test_generate_sweep_configs.py | 30 +++++++++++++------ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 2ba3d8923d..5fd8b94601 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e14c50c66b..5272fe78e6 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,7 +24,9 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 4 +# One task/Slurm allocation per concurrency gives each benchmark point a +# fresh server deployment, matching single-node agentic sweep isolation. +MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 # 3 TB decimal DRAM cap, expressed in MiB, before utilization scaling. diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 9a1c67d86e..3ab912e82c 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2253,8 +2253,8 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf with pytest.raises(ValueError, match="exceeds gpus-per-node"): generate_test_config_sweep(args, config, runner_config) - def test_multinode_agentic_groups_concurrencies_per_search_entry(self): - """One server allocation should run the selected concurrency batch.""" + def test_multinode_agentic_uses_one_allocation_per_concurrency(self): + """Each concurrency should get its own server allocation.""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2292,17 +2292,27 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): result = generate_test_config_sweep(args, config) - assert len(result) == 2 - assert result[0]["conc"] == [16, 32, 64, 128] - assert result[0]["exp-name"] == "dsv4_p2x4_d1x4_conc16x32x64x128" + assert len(result) == 5 + assert [entry["conc"] for entry in result] == [ + [16], + [32], + [64], + [128], + [256], + ] + assert [entry["exp-name"] for entry in result] == [ + "dsv4_p2x4_d1x4_conc16", + "dsv4_p2x4_d1x4_conc32", + "dsv4_p2x4_d1x4_conc64", + "dsv4_p2x4_d1x4_conc128", + "dsv4_p2x4_d1x4_conc256", + ] assert result[0]["prefill"]["pp"] == 2 assert result[0]["prefill"]["dcp-size"] == 2 assert result[0]["prefill"]["pcp-size"] == 2 assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert result[1]["conc"] == [256] - assert result[1]["exp-name"] == "dsv4_p2x4_d1x4_conc256" assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) assert all(entry["kv-p2p-transfer"] == "nixl" for entry in result) @@ -2493,9 +2503,11 @@ def test_node_type_filters_apply_to_agentic_configs( assert "prefill" not in single_result[0] assert single_result[0]["runner"] == "cluster:b300-nv" assert single_result[0]["pp"] == 2 - assert len(multi_result) == 1 - assert "prefill" in multi_result[0] + assert len(multi_result) == 2 + assert [entry["conc"] for entry in multi_result] == [[16], [32]] + assert all("prefill" in entry for entry in multi_result) assert multi_result[0]["runner"] == "cluster:gb200-nv" + assert multi_result[1]["runner"] == multi_result[0]["runner"] assert ( multi_result[0]["prefill"]["pp"], multi_result[0]["prefill"]["dcp-size"], From a5a987607813a2c556435e53fe6f17501d6e3e9f Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 09:10:11 +0000 Subject: [PATCH 02/14] agetnx: gb300 pd configs --- .../agentic/agg-gb300-tp4-agentic.yaml | 142 +++++++++++++ .../agentic/agg-gb300-tp8-agentic.yaml | 142 +++++++++++++ .../disagg-gb300-1p1d-dep8-dep8-agentic.yaml | 194 ++++++++++++++++++ .../disagg-gb300-2p1d-dep8-dep8-agentic.yaml | 192 +++++++++++++++++ .../disagg-gb300-3p1d-dep8-dep16-agentic.yaml | 192 +++++++++++++++++ configs/nvidia-master.yaml | 129 ++++++++++++ perf-changelog.yaml | 11 + runners/launch_gb300-nv.sh | 41 +++- 8 files changed, 1032 insertions(+), 11 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml new file mode 100644 index 0000000000..631d41bf4a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -0,0 +1,142 @@ +name: "svf-vllm-agg-gb300-tp4-agentic" + +# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe on one four-GPU GB300 +# node. The worker serves both prefill and decode with TP4. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml new file mode 100644 index 0000000000..1b695d4aec --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -0,0 +1,142 @@ +name: "svf-vllm-agg-gb300-tp8-agentic" + +# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe spanning two four-GPU +# GB300 nodes. The worker serves both prefill and decode with TP8. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.95 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml new file mode 100644 index 0000000000..f58ce3fc3a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml @@ -0,0 +1,194 @@ +name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-agentic" + +# GB300 AgentX reference from NVIDIA/srt-slurm#229: one DEP8 prefill worker +# feeding one DEP8 decode worker at concurrency 512. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + TORCH_SYMMMEM: "NVSHMEM" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml new file mode 100644 index 0000000000..18fca5e9a4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep8-agentic" + +# GB300 AgentX reference from NVIDIA/srt-slurm#229: two DEP8 prefill workers +# feeding one DEP8 decode worker at concurrency 1280. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.92 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 512 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml new file mode 100644 index 0000000000..7f8c7843ab --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml @@ -0,0 +1,192 @@ +name: "svf-vllm-disagg-gb300-3p1d-dep8-dep16-agentic" + +# GB300 AgentX reference from NVIDIA/srt-slurm#229: three DEP8 prefill workers +# feeding one DEP16 decode worker at concurrency 1536. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 6 + decode_nodes: 4 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + dp_launch_mode: per_node + connector: null + kv_events_config: + prefill: true + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "" + mode: "embedded" + enable_offload: false + prefill_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + decode_environment: + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3aac1b8378..f0aec91f31 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,6 +7663,135 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb300-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1, 4, 8] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [1, 8, 16] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + +dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [256, 512] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [960, 1280] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [1280, 1536] + prefill: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 005967fc36..4921fc27b8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4966,3 +4966,14 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8 + - dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8 + - dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16 + description: + - "Add GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." + - "Add NVIDIA/srt-slurm#229-derived GB300 P/D topologies: 1P/1D DEP8/DEP8 at c512, 2P/1D DEP8/DEP8 at c1280, and 3P/1D DEP8/DEP16 at c1536." + - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86d..062fb29448 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -150,9 +150,8 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic elif [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, - # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag - # (`srtctl apply --no-preflight`) — needed because: + # Agentic multi-node pins NVIDIA/srt-slurm v1.0.27, whose + # `srtctl apply --no-preflight` flag is needed because: # # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local # NVMe, fast) instead of the NFS path under /data/home/sa-shared. @@ -167,18 +166,38 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then # vLLM still fails loudly at runtime if the path is genuinely # missing on the compute node. # - # All other upstream schema features we need are inherited from - # NVIDIA HEAD: + # Other required schema features are also present in v1.0.27: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) + # - DynamoConfig.hash (so vLLM recipes can pin the ai-dynamo source) # - sbatch_directives / srun_options (top-level recipe fields) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + SRT_SLURM_AGENTIC_SHA="f6eb42aee4664207dcf2ec601e3bd57bd527efd6" + git clone --branch v1.0.27 --depth 1 https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - # 854b3fd = --no-preflight flag - # 6e34b8b = benchmark_stage propagates srun_options (needed for - # container-remap-root to reach the agentic_srt.sh srun) - git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_AGENTIC_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm v1.0.27 did not resolve to $SRT_SLURM_AGENTIC_SHA" >&2 + exit 1 + fi + + # Backport NVIDIA/srt-slurm#90. GB300 P/D workers use one vLLM process + # per physical node, with that process managing all node-local DP ranks. + SRT_SLURM_PER_NODE_DP_SHA="1a0f9e3633318ab1ee9428d2129161b583786b18" + git fetch --depth 2 origin refs/pull/90/head + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_PER_NODE_DP_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm PR #90 commit did not resolve to $SRT_SLURM_PER_NODE_DP_SHA" >&2 + exit 1 + fi + git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" + + # Multi-node TP8 needs distinct internal ZMQ ports for its node-local + # vLLM ranks rather than the inherited process-level VLLM_PORT. + SRT_SLURM_MULTINODE_VLLM_PORT_SHA="de1a4f0257dae5bf871881dc4696e35389c37483" + git fetch --depth 2 origin "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm multi-node VLLM_PORT fix did not resolve to $SRT_SLURM_MULTINODE_VLLM_PORT_SHA" >&2 + exit 1 + fi + git cherry-pick --no-commit "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" mkdir -p recipes/vllm/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic From 75d8f4477b3a8b124e61bb9d4861dd724d0ee724 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 19:02:08 +0000 Subject: [PATCH 03/14] fix(gb300): correct vLLM worker startup checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backport per-node Dynamo health counting and provide explicit GB300 NUMA mappings for aggregate workers. 中文：回移按节点启动模式下的 Dynamo 健康检查计数修复，并为 GB300 聚合式工作进程显式配置 NUMA 映射。 --- .../agentic/agg-gb300-tp4-agentic.yaml | 1 + .../agentic/agg-gb300-tp8-agentic.yaml | 1 + .../srt-slurm-vllm-per-node-health.patch | 63 +++++++++++++++++++ runners/launch_gb300-nv.sh | 8 +++ 4 files changed, 73 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 631d41bf4a..511212baf2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -104,6 +104,7 @@ backend: pipeline-parallel-size: 1 enable-cumem-allocator: true numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 32 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml index 1b695d4aec..86de563615 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -104,6 +104,7 @@ backend: pipeline-parallel-size: 1 enable-cumem-allocator: true numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 32 diff --git a/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch new file mode 100644 index 0000000000..d42553fb61 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch @@ -0,0 +1,63 @@ +diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py +index fe6cbf1..f282fe6 100644 +--- a/src/srtctl/cli/mixins/benchmark_stage.py ++++ b/src/srtctl/cli/mixins/benchmark_stage.py +@@ -46,12 +46,32 @@ def _vllm_data_parallel_size(config: "SrtConfig", mode: str) -> int: + return int(mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size") or 1) + + +-def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: ++def _vllm_health_entries( ++ config: "SrtConfig", ++ mode: str, ++ logical_workers: int, ++ backend_processes: list["Process"] | None, ++) -> int: ++ """Return expected Dynamo generate registrations for a vLLM worker mode.""" ++ dp_size = _vllm_data_parallel_size(config, mode) ++ if dp_size > 1 and getattr(config.backend, "dp_launch_mode", "per_gpu") == "per_node": ++ if backend_processes is None: ++ raise ValueError("backend_processes are required for per-node DP health expectations") ++ endpoint_mode = "agg" if mode == "aggregated" else mode ++ return sum(process.endpoint_mode == endpoint_mode for process in backend_processes) ++ ++ return logical_workers * dp_size ++ ++ ++def _get_health_expectations( ++ config: "SrtConfig", backend_processes: list["Process"] | None = None ++) -> tuple[int, int, str, int]: + """Compute expected health counts in the units reported by the frontend. + + Dynamo's /health endpoint reports registered generate instances. For vLLM +- DP workers, that means one entry per DP rank, not one entry per logical +- srt-slurm worker. Other frontends keep using logical worker counts. ++ DP workers, per-GPU launch registers one entry per DP rank, while per-node ++ launch registers one entry per node-local process. Other frontends keep ++ using logical worker counts. + """ + r = config.resources + +@@ -67,10 +87,10 @@ def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: + if config.frontend.type == "dynamo" and getattr(config.backend, "type", None) == "vllm": + if r.num_agg > 0: + n_prefill = 0 +- n_decode = logical_decode * _vllm_data_parallel_size(config, "aggregated") ++ n_decode = _vllm_health_entries(config, "aggregated", logical_decode, backend_processes) + else: +- n_prefill = logical_prefill * _vllm_data_parallel_size(config, "prefill") +- n_decode = logical_decode * _vllm_data_parallel_size(config, "decode") ++ n_prefill = _vllm_health_entries(config, "prefill", logical_prefill, backend_processes) ++ n_decode = _vllm_health_entries(config, "decode", logical_decode, backend_processes) + + count_desc = f"{n_prefill}P + {n_decode}D Dynamo generate instances; logical workers: {worker_desc}" + return n_prefill, n_decode, count_desc, n_prefill + n_decode +@@ -131,7 +151,7 @@ class BenchmarkStageMixin: + """Run the benchmark.""" + logger.info("Waiting for workers to be ready...") + +- n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config) ++ n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config, self.backend_processes) + logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) + + hc = self.config.health_check diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 062fb29448..8391ee3052 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -198,6 +198,14 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then exit 1 fi git cherry-pick --no-commit "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" + + # Per-node DP launches one Dynamo generate endpoint per node-local process, + # not one per DP rank. Backport the health-count fix from + # ivanium/srt-slurm@ca0880138fa606130ae4acbb8d0afddfb84c69fa. + SRT_SLURM_PER_NODE_HEALTH_PATCH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch" + git apply --check "$SRT_SLURM_PER_NODE_HEALTH_PATCH" + git apply "$SRT_SLURM_PER_NODE_HEALTH_PATCH" + mkdir -p recipes/vllm/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic From 9d6f3e3265a106eea771e46016c364f7ae53ffe7 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 19:24:58 +0000 Subject: [PATCH 04/14] chore: update changelog PR link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将性能变更日志中的 PR 链接更新为 #2269。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4921fc27b8..dbf631c895 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4976,4 +4976,4 @@ - "Add GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." - "Add NVIDIA/srt-slurm#229-derived GB300 P/D topologies: 1P/1D DEP8/DEP8 at c512, 2P/1D DEP8/DEP8 at c1280, and 3P/1D DEP8/DEP16 at c1536." - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2260 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2269 From ab8a655a76b28b3af76d708d88c191aca2e02cf9 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 18 Jul 2026 21:24:28 +0000 Subject: [PATCH 05/14] fix: tp configs and dynamo version --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 1 + .../vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml | 1 + .../agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml | 4 ++-- .../agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml | 4 ++-- .../agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 6 +++--- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 511212baf2..278d9d0900 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -73,6 +73,7 @@ backend: VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml index 86de563615..b78acdc154 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -73,6 +73,7 @@ backend: VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml index f58ce3fc3a..97716f4e6d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml @@ -14,11 +14,11 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" frameworks: - dynamo: "1.2.1" + dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" vllm: "0.17.2rc1.dev3675+gc188b96eb" dynamo: - hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml index 18fca5e9a4..34de79fbf5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml @@ -14,11 +14,11 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" frameworks: - dynamo: "1.2.1" + dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" vllm: "0.17.2rc1.dev3675+gc188b96eb" dynamo: - hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" install: true setup_script: vllm-container-deps.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml index 7f8c7843ab..9418824938 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml @@ -14,11 +14,11 @@ identity: container: image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" frameworks: - dynamo: "1.2.1" + dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" vllm: "0.17.2rc1.dev3675+gc188b96eb" dynamo: - hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" install: true setup_script: vllm-container-deps.sh diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f0aec91f31..4de9fcb498 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7712,7 +7712,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7741,7 +7741,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } kv-p2p-transfer: nixl multinode: true disagg: true @@ -7770,7 +7770,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16: runner: cluster:gb300-nv precision: fp4 framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } + router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } kv-p2p-transfer: nixl multinode: true disagg: true From ac0555b4c1c443cda26eb0a915f21fb2a483e210 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 04:04:39 +0000 Subject: [PATCH 06/14] fix: tp symm mem; pending PR #48843 to reenable --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 2 +- .../vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 278d9d0900..3d2f3990f8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -72,7 +72,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml index b78acdc154..5d3671593a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -72,7 +72,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" TORCH_SYMMMEM: "NVSHMEM" NCCL_CUMEM_ENABLE: "1" From eb4e2d4a524b39e639403f91473fe801705c9b1b Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 07:37:03 +0000 Subject: [PATCH 07/14] fix: tp oom --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 3d2f3990f8..7e983ded3d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -109,7 +109,7 @@ backend: attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 32 - max-num-batched-tokens: 32768 + max-num-batched-tokens: 16384 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 From f3d68f89f444b6fa22e3f3c7fd81a91dc85cad10 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 09:47:26 +0000 Subject: [PATCH 08/14] fix(gb300): reserve TP4 aggregate memory headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower gpu-memory-utilization from 0.95 to 0.94 after TP4 aggregate c1 and c8 both ran out of memory during FP4 MoE execution. 中文：将 gpu-memory-utilization 从 0.95 降至 0.94，为 GB300 TP4 聚合式 worker 预留显存余量；此前 c1 和 c8 均在执行 FP4 MoE 时发生显存不足。 --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 7e983ded3d..825d5c81f8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -115,7 +115,7 @@ backend: block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 From c4753334c366dadb461737a550023809460e5109 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 21:10:45 +0000 Subject: [PATCH 09/14] fix: pin mk device name list --- .../vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 2 +- .../vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml | 2 +- .../agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml | 2 +- .../agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml | 2 +- .../agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 825d5c81f8..95d4ffc884 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -63,7 +63,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false aggregated_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml index 5d3671593a..089fbd63bc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -63,7 +63,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false aggregated_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml index 97716f4e6d..ff95dbc812 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml @@ -71,7 +71,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml index 34de79fbf5..b4ae0a1e4f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml @@ -71,7 +71,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml index 9418824938..157047e468 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml @@ -71,7 +71,7 @@ backend: global_segment_size: "150GB" local_buffer_size: "4GB" protocol: "rdma" - device_name: "" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" mode: "embedded" enable_offload: false prefill_environment: From 5defcd8d2c3da22341210d445e3e9110fc2f4162 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 22:55:40 +0000 Subject: [PATCH 10/14] test(gb300): isolate aggregate TP validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lower TP4 and TP8 gpu-memory-utilization to 0.92, enable the DeepSeek-V4 tool-call and reasoning parsers, and limit the changelog trigger to aggregate TP configurations. 中文：将 TP4 和 TP8 的 gpu-memory-utilization 降至 0.92，启用 DeepSeek-V4 工具调用与推理解析器，并将变更日志触发范围限制为聚合式 TP 配置。 --- .../deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml | 5 ++++- .../deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml | 5 ++++- perf-changelog.yaml | 10 +++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml index 95d4ffc884..30d5733f33 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -115,10 +115,13 @@ backend: block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.94 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 + dyn-tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + dyn-reasoning-parser: deepseek_v4 sbatch_directives: cpus-per-task: "72" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml index 089fbd63bc..ade5e4acef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -115,10 +115,13 @@ backend: block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 32 - gpu-memory-utilization: 0.95 + gpu-memory-utilization: 0.92 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: deepseek_v4 + dyn-tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + dyn-reasoning-parser: deepseek_v4 sbatch_directives: cpus-per-task: "72" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index dbf631c895..1fffc16a0e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4969,11 +4969,11 @@ - config-keys: - dsv4-fp4-gb300-dynamo-vllm-agentic-agg - - dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8 - - dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8 - - dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16 +# - dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8 +# - dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8 +# - dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - - "Add GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." - - "Add NVIDIA/srt-slurm#229-derived GB300 P/D topologies: 1P/1D DEP8/DEP8 at c512, 2P/1D DEP8/DEP8 at c1280, and 3P/1D DEP8/DEP16 at c1536." + - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." + - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers." - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2269 From 450e17dd92808fad7d88f2ab5c6d8a09e28c06a1 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 22:56:15 +0000 Subject: [PATCH 11/14] chore: link GB300 TP test PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point the isolated aggregate TP changelog entry at draft PR #2284. 中文：将独立的聚合式 TP 变更日志条目指向草稿 PR #2284。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1fffc16a0e..b541a0beb2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4976,4 +4976,4 @@ - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers." - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2269 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2284 From d176a40539979a07c2796f52179536106412e652 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 19 Jul 2026 23:21:22 +0000 Subject: [PATCH 12/14] test(gb300): remove disaggregated test scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the P/D recipes, master-config entries, changelog references, and per-node-DP launcher support so draft PR #2284 validates only aggregate TP4 and TP8. 中文：移除 P/D 推理配置、主配置条目、变更日志引用以及单节点内 DP 启动支持，使草稿 PR #2284 仅验证聚合式 TP4 和 TP8。 --- .../disagg-gb300-1p1d-dep8-dep8-agentic.yaml | 194 ------------------ .../disagg-gb300-2p1d-dep8-dep8-agentic.yaml | 192 ----------------- .../disagg-gb300-3p1d-dep8-dep16-agentic.yaml | 192 ----------------- .../srt-slurm-vllm-per-node-health.patch | 63 ------ configs/nvidia-master.yaml | 87 -------- perf-changelog.yaml | 3 - runners/launch_gb300-nv.sh | 17 -- 7 files changed, 748 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml deleted file mode 100644 index ff95dbc812..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml +++ /dev/null @@ -1,194 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-agentic" - -# GB300 AgentX reference from NVIDIA/srt-slurm#229: one DEP8 prefill worker -# feeding one DEP8 decode worker at concurrency 512. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - frameworks: - dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - vllm: "0.17.2rc1.dev3675+gc188b96eb" - -dynamo: - hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - decode_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - TORCH_SYMMMEM: "NVSHMEM" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml deleted file mode 100644 index b4ae0a1e4f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: "svf-vllm-disagg-gb300-2p1d-dep8-dep8-agentic" - -# GB300 AgentX reference from NVIDIA/srt-slurm#229: two DEP8 prefill workers -# feeding one DEP8 decode worker at concurrency 1280. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - frameworks: - dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - vllm: "0.17.2rc1.dev3675+gc188b96eb" - -dynamo: - hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - decode_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml deleted file mode 100644 index 157047e468..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: "svf-vllm-disagg-gb300-3p1d-dep8-dep16-agentic" - -# GB300 AgentX reference from NVIDIA/srt-slurm#229: three DEP8 prefill workers -# feeding one DEP16 decode worker at concurrency 1536. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" - frameworks: - dynamo: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - vllm: "0.17.2rc1.dev3675+gc188b96eb" - -dynamo: - hash: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - dp_launch_mode: per_node - connector: null - kv_events_config: - prefill: true - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - prefill_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - decode_environment: - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - data-parallel-hybrid-lb: true - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 128 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.90 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch b/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch deleted file mode 100644 index d42553fb61..0000000000 --- a/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch +++ /dev/null @@ -1,63 +0,0 @@ -diff --git a/src/srtctl/cli/mixins/benchmark_stage.py b/src/srtctl/cli/mixins/benchmark_stage.py -index fe6cbf1..f282fe6 100644 ---- a/src/srtctl/cli/mixins/benchmark_stage.py -+++ b/src/srtctl/cli/mixins/benchmark_stage.py -@@ -46,12 +46,32 @@ def _vllm_data_parallel_size(config: "SrtConfig", mode: str) -> int: - return int(mode_config.get("data-parallel-size") or mode_config.get("data_parallel_size") or 1) - - --def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: -+def _vllm_health_entries( -+ config: "SrtConfig", -+ mode: str, -+ logical_workers: int, -+ backend_processes: list["Process"] | None, -+) -> int: -+ """Return expected Dynamo generate registrations for a vLLM worker mode.""" -+ dp_size = _vllm_data_parallel_size(config, mode) -+ if dp_size > 1 and getattr(config.backend, "dp_launch_mode", "per_gpu") == "per_node": -+ if backend_processes is None: -+ raise ValueError("backend_processes are required for per-node DP health expectations") -+ endpoint_mode = "agg" if mode == "aggregated" else mode -+ return sum(process.endpoint_mode == endpoint_mode for process in backend_processes) -+ -+ return logical_workers * dp_size -+ -+ -+def _get_health_expectations( -+ config: "SrtConfig", backend_processes: list["Process"] | None = None -+) -> tuple[int, int, str, int]: - """Compute expected health counts in the units reported by the frontend. - - Dynamo's /health endpoint reports registered generate instances. For vLLM -- DP workers, that means one entry per DP rank, not one entry per logical -- srt-slurm worker. Other frontends keep using logical worker counts. -+ DP workers, per-GPU launch registers one entry per DP rank, while per-node -+ launch registers one entry per node-local process. Other frontends keep -+ using logical worker counts. - """ - r = config.resources - -@@ -67,10 +87,10 @@ def _get_health_expectations(config: "SrtConfig") -> tuple[int, int, str, int]: - if config.frontend.type == "dynamo" and getattr(config.backend, "type", None) == "vllm": - if r.num_agg > 0: - n_prefill = 0 -- n_decode = logical_decode * _vllm_data_parallel_size(config, "aggregated") -+ n_decode = _vllm_health_entries(config, "aggregated", logical_decode, backend_processes) - else: -- n_prefill = logical_prefill * _vllm_data_parallel_size(config, "prefill") -- n_decode = logical_decode * _vllm_data_parallel_size(config, "decode") -+ n_prefill = _vllm_health_entries(config, "prefill", logical_prefill, backend_processes) -+ n_decode = _vllm_health_entries(config, "decode", logical_decode, backend_processes) - - count_desc = f"{n_prefill}P + {n_decode}D Dynamo generate instances; logical workers: {worker_desc}" - return n_prefill, n_decode, count_desc, n_prefill + n_decode -@@ -131,7 +151,7 @@ class BenchmarkStageMixin: - """Run the benchmark.""" - logger.info("Waiting for workers to be ready...") - -- n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config) -+ n_prefill, n_decode, count_desc, num_workers = _get_health_expectations(self.config, self.backend_processes) - logger.info("Waiting for server health (expecting %d health entries: %s)...", num_workers, count_desc) - - hc = self.config.health_check diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4de9fcb498..a448a16ca3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7705,93 +7705,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-agg: ep: 1 dp-attn: false -dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [256, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [960, 1280] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "af0dbfe8233fafdfaaa5d12e17518758c99e8da5" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [1280, 1536] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-3p1d-dep8-dep16-agentic.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b541a0beb2..410a6c8218 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4969,9 +4969,6 @@ - config-keys: - dsv4-fp4-gb300-dynamo-vllm-agentic-agg -# - dsv4-fp4-gb300-dynamo-vllm-agentic-1p1d-dep8-dep8 -# - dsv4-fp4-gb300-dynamo-vllm-agentic-2p1d-dep8-dep8 -# - dsv4-fp4-gb300-dynamo-vllm-agentic-3p1d-dep8-dep16 description: - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers." diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 8391ee3052..3b88f6af70 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -179,16 +179,6 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then exit 1 fi - # Backport NVIDIA/srt-slurm#90. GB300 P/D workers use one vLLM process - # per physical node, with that process managing all node-local DP ranks. - SRT_SLURM_PER_NODE_DP_SHA="1a0f9e3633318ab1ee9428d2129161b583786b18" - git fetch --depth 2 origin refs/pull/90/head - if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_PER_NODE_DP_SHA" ]]; then - echo "Error: NVIDIA/srt-slurm PR #90 commit did not resolve to $SRT_SLURM_PER_NODE_DP_SHA" >&2 - exit 1 - fi - git cherry-pick --no-commit "$SRT_SLURM_PER_NODE_DP_SHA" - # Multi-node TP8 needs distinct internal ZMQ ports for its node-local # vLLM ranks rather than the inherited process-level VLLM_PORT. SRT_SLURM_MULTINODE_VLLM_PORT_SHA="de1a4f0257dae5bf871881dc4696e35389c37483" @@ -199,13 +189,6 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then fi git cherry-pick --no-commit "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" - # Per-node DP launches one Dynamo generate endpoint per node-local process, - # not one per DP rank. Backport the health-count fix from - # ivanium/srt-slurm@ca0880138fa606130ae4acbb8d0afddfb84c69fa. - SRT_SLURM_PER_NODE_HEALTH_PATCH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-vllm-per-node-health.patch" - git apply --check "$SRT_SLURM_PER_NODE_HEALTH_PATCH" - git apply "$SRT_SLURM_PER_NODE_HEALTH_PATCH" - mkdir -p recipes/vllm/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic From b020d3c0724e738c9e675a8280a289c971849601 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Mon, 20 Jul 2026 23:19:39 +0000 Subject: [PATCH 13/14] test(gb300): restrict TP validation points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limit the GB300 Dynamo-vLLM AgentX aggregate validation to TP4 at concurrency 4 and TP8 at concurrency 1. Update the changelog description to match. 中文：将 GB300 Dynamo-vLLM AgentX 聚合验证限制为 TP4 并发 4 和 TP8 并发 1，并同步更新变更日志说明。 --- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a448a16ca3..6683f1127e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7677,7 +7677,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-agg: agentic-coding: - search-space: - spec-decoding: none - conc-list: [1, 4, 8] + conc-list: [4] prefill: num-worker: 1 tp: 4 @@ -7691,7 +7691,7 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-agg: ep: 1 dp-attn: false - spec-decoding: none - conc-list: [1, 8, 16] + conc-list: [1] prefill: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 410a6c8218..a78b0de80f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4970,7 +4970,7 @@ - config-keys: - dsv4-fp4-gb300-dynamo-vllm-agentic-agg description: - - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4/TP8 low-latency sweeps at conc [1,4,8,16]." + - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4 at conc 4 and TP8 at conc 1." - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers." - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2284 From e106f0275adcaad2e8b7cd404a168b6a8b851d7a Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Tue, 21 Jul 2026 00:15:53 +0000 Subject: [PATCH 14/14] chore: drop redundant matrix logic changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：移除 TP 测试分支中已由上游 main 覆盖的冗余矩阵逻辑改动。 --- utils/matrix_logic/generate_sweep_configs.py | 3 +-- utils/matrix_logic/test_generate_sweep_configs.py | 15 +++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index c4c509ea3a..7df1a75b64 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -24,8 +24,7 @@ MIN_EVAL_CONC = 16 # Bound how many multinode agentic conc points share one server allocation. -# One task/Slurm allocation per concurrency gives each benchmark point a -# fresh server deployment, matching single-node agentic sweep isolation. +# 1 = one task/SLURM allocation per concurrency (matches single-node agentic). MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION = 1 BYTES_PER_MIB = 1024 * 1024 BYTES_PER_GB = 1_000_000_000 diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index a93ebdf186..1ad4e3c922 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2175,8 +2175,8 @@ def test_agentic_node_dram_rejects_tp_above_runner_gpus(self, sample_runner_conf with pytest.raises(ValueError, match="exceeds gpus-per-node"): generate_test_config_sweep(args, config, runner_config) - def test_multinode_agentic_uses_one_allocation_per_concurrency(self): - """Each concurrency should get its own server allocation.""" + def test_multinode_agentic_groups_concurrencies_per_search_entry(self): + """One server allocation should run exactly one concurrency (one task per conc).""" config = { "dsv4-agentic-2p1d": { "image": "vllm/vllm-openai:v0.23.0", @@ -2187,7 +2187,6 @@ def test_multinode_agentic_uses_one_allocation_per_concurrency(self): "runner": "gb200", "multinode": True, "disagg": True, - "router": {"name": "dynamo-router", "version": "1.3.0"}, "kv-p2p-transfer": "nixl", "scenarios": { "agentic-coding": [ @@ -2215,13 +2214,7 @@ def test_multinode_agentic_uses_one_allocation_per_concurrency(self): result = generate_test_config_sweep(args, config) assert len(result) == 5 - assert [entry["conc"] for entry in result] == [ - [16], - [32], - [64], - [128], - [256], - ] + assert [entry["conc"] for entry in result] == [[16], [32], [64], [128], [256]] assert [entry["exp-name"] for entry in result] == [ "dsv4_p2x4_d1x4_conc16", "dsv4_p2x4_d1x4_conc32", @@ -2235,8 +2228,6 @@ def test_multinode_agentic_uses_one_allocation_per_concurrency(self): assert result[0]["decode"]["pp"] == 2 assert result[0]["decode"]["dcp-size"] == 2 assert result[0]["decode"]["pcp-size"] == 1 - assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) - assert all(entry["kv-p2p-transfer"] == "nixl" for entry in result) def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_config): config = {