diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 2ba3d8923d..5fd8b94601 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1797,7 +1797,11 @@ build_replay_cmd() { # X-Correlation-ID is useful tracing metadata but does not establish that # binding by itself. AIPerf emits nvext.session_control bind/close actions # keyed by the stable conversation correlation ID when this flag is set. - if [[ "${FRAMEWORK:-}" == dynamo-* ]]; then + # Opt-out: recipes set AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING=0 to skip this. + # aiperf's conv-aware routing emits nvext.session_control, a removed POC field + # (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400 + # (they moved to router/routing_constraints/agent_context). Default stays on. + if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then REPLAY_CMD+=" --use-dynamo-conv-aware-routing" # The upstream 300s affinity TTL is shorter than an overloaded # high-concurrency agentic request. Keep bindings alive across long diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml new file mode 100644 index 0000000000..30d5733f33 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml @@ -0,0 +1,147 @@ +name: "svf-vllm-agg-gb300-tp4-agentic" + +# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe on one four-GPU GB300 +# node. The worker serves both prefill and decode with TP4. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 16384 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.92 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + dyn-tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + dyn-reasoning-parser: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml new file mode 100644 index 0000000000..ade5e4acef --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml @@ -0,0 +1,147 @@ +name: "svf-vllm-agg-gb300-tp8-agentic" + +# Low-latency aggregate DeepSeek-V4-Pro AgentX recipe spanning two four-GPU +# GB300 nodes. The worker serves both prefill and decode with TP8. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96" + frameworks: + dynamo: "1.2.1" + vllm: "0.17.2rc1.dev3675+gc188b96eb" + +dynamo: + hash: "919682da679aa699d5bca9c872f4c1d9a530bbc0" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "kv" + router-reset-states: true + router-temperature: 0.0 + router-queue-threshold: 65536 + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + tokenizer: "fastokens" + +backend: + type: vllm + connector: null + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + aggregated_environment: + DYN_REQUEST_PLANE: "tcp" + ETCD_LEASE_TTL: "120" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + TILELANG_CLEANUP_TEMP_FILES: "1" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + TORCH_SYMMMEM: "NVSHMEM" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "4" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-agentx-{job_id}" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_RCACHE_MAX_UNRELEASED: "1024" + UCX_TLS: "cuda_copy,cuda_ipc,tcp" + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + + vllm_config: + aggregated: + kv-transfer-config: '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + enable-cumem-allocator: true + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 32 + max-num-batched-tokens: 32768 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 32 + gpu-memory-utilization: 0.92 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + dyn-tool-call-parser: deepseek_v4 + reasoning-parser: deepseek_v4 + dyn-reasoning-parser: deepseek_v4 + +sbatch_directives: + cpus-per-task: "72" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + EP_SIZE: "1" + DP_ATTENTION: "false" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3aac1b8378..6683f1127e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7663,6 +7663,48 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb300-dynamo-vllm-agentic-agg: + image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: none + conc-list: [4] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-agentic.yaml" + decode: + num-worker: 0 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [1] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + dsv4-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 892e3ee2b6..af41927a61 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4974,3 +4974,11 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-agg + description: + - "Test only the GB300 Dynamo-vLLM AgentX aggregate TP4 at conc 4 and TP8 at conc 1." + - "Set gpu-memory-utilization to 0.92 for TP4/TP8 and enable the DeepSeek-V4 tool-call and reasoning parsers." + - "Use vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-c188b96 for all new recipes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2284 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86d..3b88f6af70 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -150,9 +150,8 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic elif [[ "$IS_AGENTIC" == "1" ]]; then - # Agentic multi-node uses cquil11/srt-slurm-nv@cam/no-preflight-flag, - # a thin branch off NVIDIA/srt-slurm@127597c that adds one CLI flag - # (`srtctl apply --no-preflight`) — needed because: + # Agentic multi-node pins NVIDIA/srt-slurm v1.0.27, whose + # `srtctl apply --no-preflight` flag is needed because: # # - We want MODEL_PATH=/scratch/models/DeepSeek-V4-Pro (node-local # NVMe, fast) instead of the NFS path under /data/home/sa-shared. @@ -167,18 +166,29 @@ elif [[ "$IS_AGENTIC" == "1" ]]; then # vLLM still fails loudly at runtime if the path is genuinely # missing on the compute node. # - # All other upstream schema features we need are inherited from - # NVIDIA HEAD: + # Other required schema features are also present in v1.0.27: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (hook that hands off to benchmarks/multi_node/agentic_srt.sh) - # - DynamoConfig.wheel (so vllm recipes can pin the ai-dynamo wheel) + # - DynamoConfig.hash (so vLLM recipes can pin the ai-dynamo source) # - sbatch_directives / srun_options (top-level recipe fields) - git clone https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" + SRT_SLURM_AGENTIC_SHA="f6eb42aee4664207dcf2ec601e3bd57bd527efd6" + git clone --branch v1.0.27 --depth 1 https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - # 854b3fd = --no-preflight flag - # 6e34b8b = benchmark_stage propagates srun_options (needed for - # container-remap-root to reach the agentic_srt.sh srun) - git checkout 6e34b8b83229634d732e41a4e2d6595f46ef60b5 + if [[ "$(git rev-parse HEAD)" != "$SRT_SLURM_AGENTIC_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm v1.0.27 did not resolve to $SRT_SLURM_AGENTIC_SHA" >&2 + exit 1 + fi + + # Multi-node TP8 needs distinct internal ZMQ ports for its node-local + # vLLM ranks rather than the inherited process-level VLLM_PORT. + SRT_SLURM_MULTINODE_VLLM_PORT_SHA="de1a4f0257dae5bf871881dc4696e35389c37483" + git fetch --depth 2 origin "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" ]]; then + echo "Error: NVIDIA/srt-slurm multi-node VLLM_PORT fix did not resolve to $SRT_SLURM_MULTINODE_VLLM_PORT_SHA" >&2 + exit 1 + fi + git cherry-pick --no-commit "$SRT_SLURM_MULTINODE_VLLM_PORT_SHA" + mkdir -p recipes/vllm/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ recipes/vllm/deepseek-v4/agentic