diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml index 3ff9e31924..7fee5dc431 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -16,7 +16,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -44,11 +44,10 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" # The sglang image is PEP 668 externally-managed; the runtime dynamo # install (dynamo_wheels.py / source build) runs pip and fails with # "externally-managed-environment" without this. Lets pip install into @@ -56,7 +55,7 @@ frontend: PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -66,6 +65,9 @@ backend: aggregated_environment: SGLANG_DEFAULT_THINKING: '1' + # TP4/bs128 can spend over an hour capturing the DSV4 EAGLE draft-extend + # graph. Keep the target/decode graphs and use the upstream eager fallback. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" SGLANG_DSV4_REASONING_EFFORT: high SGLANG_SIMULATE_ACC_LEN: '2.49' SGLANG_SIMULATE_ACC_METHOD: match-expected @@ -139,11 +141,9 @@ benchmark: # in process_agentic_result.py instead. TP must match sglang_config tp-size. IS_MULTINODE: "false" TP: "4" - # Enable dynamo conv-aware routing (per-session affinity so multi-turn - # KV cache is reused across turns). Previously opted out because the - # frontend 400'd on aiperf's nvext.session_control actions; re-enabled - # to test with the current build. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + # Stamp X-Dynamo-Session-ID on every turn so both nginx and Dynamo's + # router keep the session on the worker that owns its KV prefix. + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" # Container-side path of the aiperf mmap dataset cache; host-side mount # wired via launch_gb300-*.sh srtslurm.yaml default_mounts. Without it, # aiperf re-tokenizes + re-writes the dataset mmap on every run. @@ -151,4 +151,3 @@ benchmark: # Persistent HF hub cache (also via default_mounts) so the trace dataset # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml index 715b0c631e..1e00283043 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -206,7 +205,6 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml index dc91c255ce..d926b9a937 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -92,6 +91,9 @@ backend: decode_environment: SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + # Match the TP4 aggregate worker: avoid the pathological DSV4 EAGLE + # draft-extend graph capture while retaining the target/decode graphs. + SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH: "1" SGLANG_SIMULATE_ACC_LEN: '2.49' SGLANG_SIMULATE_ACC_METHOD: match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" @@ -205,7 +207,6 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 6df695b914..0c9df6f9ec 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -206,7 +205,6 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 107a787ba9..3fbe5268eb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -40,15 +40,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -206,7 +205,6 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 27a4e42d22..4b17f493ef 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -15,7 +15,7 @@ model: dynamo: install: true - wheel: "1.3.0.dev1" + wheel: "1.3.0.dev20260718" slurm: time_limit: "8:00:00" @@ -41,15 +41,14 @@ infra: frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Correlation-ID + nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - DYN_ROUTER_TEMPERATURE: "10000000" PIP_BREAK_SYSTEM_PACKAGES: "1" args: router-mode: "kv" - router-reset-states: true + router-session-affinity-ttl-secs: "3600" active-decode-blocks-threshold: "None" active-prefill-tokens-threshold: "None" active-prefill-tokens-threshold-frac: "None" @@ -206,7 +205,6 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..f8eeb69cbc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8836,13 +8836,13 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev1" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } multinode: true disagg: false scenarios: @@ -8870,13 +8870,13 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: - image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + image: lmsysorg/sglang:nightly-dev-cu13-20260719-99f5a6f4 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "1.3.0.dev1" } + router: { name: dynamo-router, version: "1.3.0.dev20260718" } kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..a27a274c4b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,13 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Upgrade Dynamo from 1.3.0.dev1 to 1.3.0.dev20260718 so router queueing is disabled by default at high concurrency" + - "Use AIPerf f5ae8134 (including the X-Dynamo-Session-ID session-routing support merged in AIPerf #19) and migrate AgentX session affinity from nvext.session_control to X-Dynamo-Session-ID headers with a 3600-second router TTL" + - "Remove the DYN_ROUTER_TEMPERATURE=10000000 override and restore Dynamo's deterministic 0.0 default" + - "Upgrade SGLang from nightly-dev-cu13-20260711-7de33ce8 to nightly-dev-cu13-20260719-99f5a6f4, the latest nightly before the scheduler WAR-barrier regression in SGLang #31687" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2319