From 4ac75e3b790c3eb2f53a96a4b7edebc1aa9fee6d Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 20 Jul 2026 03:14:30 +0000 Subject: [PATCH 1/8] [AMD][dsv4] fix 2P1D eval 503s and 8k1k straggler tail - Speed circuit-breaker recovery and gate benchmarking on an end-to-end router readiness canary, so a transient trip or a not-yet-serving router doesn't 503 every request until the client's retry budget is exhausted (port of PR #2255). - Keep decode's admission context-length symmetric with prefill (falling back to prefill's value when unset in models.yaml), so over-length requests are rejected fast by both sides instead of hanging on decode while waiting for a KV transfer prefill never sends (port of PR #2257). Co-authored-by: Cursor Co-authored-by: inkcherry --- benchmarks/multi_node/amd_utils/models.yaml | 9 +++ .../multi_node/amd_utils/server_sglang.sh | 61 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index bc2b39aa07..bc80c7d21d 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -395,6 +395,15 @@ DeepSeek-V4-Pro: dp: max_running_requests: 1024 cuda_graph_bs_range: "1-128" + # Must match prefill.context_length. In PD-disagg the decode admits requests + # against its own context length; if left unset it defaults to the model's + # (huge) config value, so a request whose input exceeds prefill's 9217 budget + # is accepted by decode (pre-allocated, waiting for KV) but rejected by prefill + # (never sends KV) -> the request hangs to the client timeout (the 8k1k + # conc-500 straggler tail). Keeping decode symmetric with prefill makes such + # over-length requests fail fast at admission instead. + context_length: 9217 no_dp: max_running_requests: 128 cuda_graph_bs_range: "1-128" + context_length: 9217 diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index ec5805eb0e..e6c8b9fcb3 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -154,6 +154,7 @@ no_dp = decode.get('no_dp', {}) # Decode DP config print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') @@ -161,6 +162,7 @@ print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') # Decode EP-only config (EP enabled but DP disabled) print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"') s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') @@ -168,6 +170,7 @@ print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') # Decode no-DP config print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') @@ -205,12 +208,23 @@ fi if [[ "$DECODE_ENABLE_DP" == "true" ]]; then decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) + decode_context_length=$DECODE_CONTEXT_LENGTH_DP elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY + decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY else decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP + decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP +fi +# In PD-disaggregation the decode must admit requests against the SAME context +# length as prefill; otherwise decode accepts over-length requests that prefill +# rejects, and those requests hang forever waiting for a KV transfer that never +# comes (the 8k1k conc-500 straggler). Fall back to the prefill value if the +# decode context_length is not set in the model config, so the two always agree. +if [[ -z "$decode_context_length" ]]; then + decode_context_length=$prefill_context_length fi # When both DP and EP are enabled, override max-running-requests and dispatch tokens @@ -251,6 +265,9 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" fi +if [[ -n "$decode_context_length" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}" +fi if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) @@ -506,12 +523,23 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "Congratulations!!! All prefill and decode servers are up . . ." + # Circuit-breaker recovery tuning (run 28696443568): + # With defaults the per-worker circuit stays OPEN for cb-timeout-duration-secs=60 + # before a half-open retrial. In that run every request 503'd from request #1 + # ("all circuits open or unhealthy") for ~31s and lm_eval (max_retries=5) then gave + # up -- i.e. the circuit was still open when the client budget ran out, so 0 result + # files were produced. Shortening the open->half-open window (and letting the router + # itself retry a failed worker selection) lets a transient trip re-close INSIDE the + # client retry budget instead of nuking the whole eval. The breaker stays fully + # ENABLED (thresholds unchanged); this only speeds recovery. + ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}" ROUTER_CMD="python -m sglang_router.launch_router \ --pd-disaggregation \ --port 30000 \ --policy random \ --prefill-policy random \ --decode-policy random \ + ${ROUTER_CB_ARGS} \ ${PREFILL_ARGS} \ ${DECODE_ARGS}" @@ -557,6 +585,39 @@ if [ "$NODE_RANK" -eq 0 ]; then wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1 fi + # ---- End-to-end router readiness canary (run 28696443568) ---- + # The /readiness barrier above only proves the router PROCESS is up; it does + # NOT prove the router can reach a prefill worker and complete a generation. + # In that run the eval fired the instant /readiness passed and EVERY request + # 503'd ("No available prefill workers (all circuits open or unhealthy)") from + # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed. + # Gate the benchmark on ONE successful generation THROUGH the router so the eval + # never starts against a router whose prefill path is not yet actually serving. + if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then + CANARY_URL="http://${NODE0_ADDR}:30000/v1/chat/completions" + CANARY_MODEL="${MODEL_DIR}/${MODEL_NAME}" + canary_ok=0 + canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) + while [ "$(date +%s)" -lt "$canary_deadline" ]; do + canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ + -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \ + -X POST "$CANARY_URL" -H 'Content-Type: application/json' \ + -d "{\"model\":\"${CANARY_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) + if [ "$canary_code" = "200" ] && \ + ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then + canary_ok=1; break + fi + echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." + sleep 5 + done + if [ "$canary_ok" -ne 1 ]; then + echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." + head -c 800 /tmp/router_canary.out 2>/dev/null + exit 1 + fi + echo "Router readiness canary passed (end-to-end generation OK)" + fi + echo "Router is ready for benchmarking" fi From 6804a4c92bbb80850addb376ee0999c327b26ffc Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 20 Jul 2026 03:21:54 +0000 Subject: [PATCH 2/8] add con=500/512 Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d829459643..fe6cd0f1c4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1169,7 +1169,7 @@ dsv4-fp4-mi355x-sglang-disagg: # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 128, 256 ] + conc-list: [ 500, 512 ] prefill: num-worker: 2 tp: 8 From ef481e4ede9ea0d382c33e4081f24aa499fded66 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 20 Jul 2026 09:57:59 +0000 Subject: [PATCH 3/8] set max_total_tokens: 2097152 Signed-off-by: Theresa Shan Co-authored-by: lixiufei-leo --- benchmarks/multi_node/amd_utils/models.yaml | 4 +- configs/amd-master.yaml | 55 +++++++++++++++++---- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index bc80c7d21d..8b131461ec 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -383,12 +383,12 @@ DeepSeek-V4-Pro: max_running_requests: 1024 chunked_prefill_size: 131072 context_length: 9217 - max_total_tokens: 262144 + max_total_tokens: 2097152 no_dp: max_running_requests: 128 chunked_prefill_size: 131072 context_length: 9217 - max_total_tokens: 262144 + max_total_tokens: 2097152 decode: mem_fraction_static: 0.85 prefill_round_robin_balance: true diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index fe6cd0f1c4..10da9c8f41 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1149,34 +1149,69 @@ dsv4-fp4-mi355x-sglang-disagg: search-space: # non-MTP configurations # 1P1D pure TP8 (mori KV transfer) + # - spec-decoding: "none" + # conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] + # prefill: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "PREFILL_NODES=1" + # decode: + # num-worker: 1 + # tp: 8 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "DECODE_NODES=1" + # - "DECODE_MTP_SIZE=0" + + # 2P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] + conc-list: [ 256, 512, 1024 ] prefill: - num-worker: 1 + num-worker: 2 tp: 8 - ep: 1 - dp-attn: false + ep: 8 + dp-attn: true additional-settings: - - "PREFILL_NODES=1" + - "PREFILL_NODES=2" decode: num-worker: 1 tp: 8 - ep: 1 - dp-attn: false + ep: 8 + dp-attn: true additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" +dsv4-fp4-mi355x-sglang-disagg-1p1d: + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: mi355x-disagg + precision: fp4 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 500, 512 ] + conc-list: [ 256, 512, 1024 ] prefill: - num-worker: 2 + num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" + - "PREFILL_NODES=1" decode: num-worker: 1 tp: 8 From 78a3345d6af5ee179841eaae9c185cb72d44790d Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Mon, 20 Jul 2026 10:09:08 +0000 Subject: [PATCH 4/8] update config Signed-off-by: Theresa Shan --- configs/amd-master.yaml | 111 ++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 38 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 10da9c8f41..6bb375378f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1147,29 +1147,45 @@ dsv4-fp4-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # non-MTP configurations - # 1P1D pure TP8 (mori KV transfer) - # - spec-decoding: "none" - # conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - # prefill: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "PREFILL_NODES=1" - # decode: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "DECODE_NODES=1" - # - "DECODE_MTP_SIZE=0" - # 2P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 256, 512, 1024 ] + conc-list: [ 256 ] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + - spec-decoding: "none" + conc-list: [ 512 ] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + - spec-decoding: "none" + conc-list: [ 1024 ] prefill: num-worker: 2 tp: 8 @@ -1186,25 +1202,9 @@ dsv4-fp4-mi355x-sglang-disagg: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" -dsv4-fp4-mi355x-sglang-disagg-1p1d: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 256, 512, 1024 ] + conc-list: [ 256 ] prefill: num-worker: 1 tp: 8 @@ -1221,6 +1221,41 @@ dsv4-fp4-mi355x-sglang-disagg-1p1d: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" + - spec-decoding: "none" + conc-list: [ 512 ] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + - spec-decoding: "none" + conc-list: [ 1024 ] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 From f392a2ca3adf44327dcdb3d31e14d2b848b9cf33 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 21 Jul 2026 07:40:37 +0000 Subject: [PATCH 5/8] [AMD][dsv4] Replace 2P1D DEP8 sweep with 1P1D pure-TP8 in disagg non-MTP search space Drop the 2P1D DEP8 (conc 256/512/1024) and 1P1D DEP8 conc=1024 entries and add a 1P1D pure-TP8 (no EP, no DP-attention) scenario sweeping conc [1, 2, 4, 8, 16, 32, 64, 128] for dsv4-fp4-mi355x-sglang-disagg. Co-authored-by: Cursor --- configs/amd-master.yaml | 71 ++++++----------------------------------- 1 file changed, 9 insertions(+), 62 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6bb375378f..9e36a1ecc4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1147,61 +1147,25 @@ dsv4-fp4-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # 2P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "none" - conc-list: [ 256 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - + # non-MTP configurations + # 1P1D pure TP8 (mori KV transfer) - spec-decoding: "none" - conc-list: [ 512 ] + conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: num-worker: 1 tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 1024 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "PREFILL_NODES=2" + - "PREFILL_NODES=1" decode: num-worker: 1 tp: 8 - ep: 8 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" conc-list: [ 256 ] @@ -1238,24 +1202,7 @@ dsv4-fp4-mi355x-sglang-disagg: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" + dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 From f5d3961907679adf7f54a073e7a1cccc25b58c48 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 21 Jul 2026 08:46:53 +0000 Subject: [PATCH 6/8] remove decode context_length Squash of f7295a53f "remove decode context_length", 0be1d3014 "Revert 'remove decode context_length' for DeepSeek-V4-Pro", and bfb898828 "remove decode context_length ending with a newline" into a single commit with the same net result. Signed-off-by: Theresa Shan Co-authored-by: Cursor --- benchmarks/multi_node/amd_utils/models.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 8b131461ec..c37a4062b2 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -395,15 +395,6 @@ DeepSeek-V4-Pro: dp: max_running_requests: 1024 cuda_graph_bs_range: "1-128" - # Must match prefill.context_length. In PD-disagg the decode admits requests - # against its own context length; if left unset it defaults to the model's - # (huge) config value, so a request whose input exceeds prefill's 9217 budget - # is accepted by decode (pre-allocated, waiting for KV) but rejected by prefill - # (never sends KV) -> the request hangs to the client timeout (the 8k1k - # conc-500 straggler tail). Keeping decode symmetric with prefill makes such - # over-length requests fail fast at admission instead. - context_length: 9217 no_dp: max_running_requests: 128 cuda_graph_bs_range: "1-128" - context_length: 9217 From 4b3ac7c2fe3cb25e5c73692f0da9a42511006394 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 21 Jul 2026 09:07:02 +0000 Subject: [PATCH 7/8] Add perf-changelog.yaml entry for dsv4-fp4-mi355x-sglang-disagg retune Per the automated PR reviewer: any edit to configs/amd-master.yaml requires a paired perf-changelog.yaml entry. Documents the 503/ straggler-tail fixes, the max_total_tokens bump, and the 2P1D DEP8 -> 1P1D DEP8/pure-TP8 concurrency sweep retune from PR #2293. Co-authored-by: Cursor --- perf-changelog.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 005967fc36..c050ae0489 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4966,3 +4966,13 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg + description: + - "Retune the non-MTP 8k1k disagg search space: drop the 2P1D DEP8 sweep (conc 128/256/512/1024 across 2 prefill nodes) and the 1P1D DEP8 conc=1024 point" + - "Add 1P1D DEP8 (TP8/EP8 + dp-attention, 1 prefill node) at conc 256/512, and a new 1P1D pure-TP8 (no EP, no DP-attention) scenario sweeping conc [1, 2, 4, 8, 16, 32, 64, 128]" + - "Fix 2P1D eval 503s: speed up router circuit-breaker recovery (--cb-timeout-duration-secs 15 --retry-max-retries 3) and gate benchmarking on an end-to-end router readiness canary (one real generation through the router), so the eval no longer starts against a router whose prefill path isn't actually serving yet" + - "Fix the 8k1k conc-500 straggler tail: decode now derives its admission context_length from models.yaml (falling back to prefill's value when unset) so decode and prefill reject over-length requests symmetrically instead of decode hanging on a KV transfer prefill never sends" + - "Bump DeepSeek-V4-Pro prefill max_total_tokens from 262144 to 2097152 to match the model's actual KV budget" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2293 From 208bff469e32c97d5fadaf8ae9999bea7ae0c298 Mon Sep 17 00:00:00 2001 From: Theresa Shan Date: Tue, 21 Jul 2026 10:19:19 +0000 Subject: [PATCH 8/8] Guard router readiness canary with wait_or_die on prefill0_pid The canary poll loop was a plain while/curl loop, unlike every other blocking wait on node rank 0 (container barrier, servers-up barrier, /readiness health barrier), so a prefill crash right after /readiness passed just looked like repeated 503s and burned the full ROUTER_CANARY_TIMEOUT (default 600s) instead of failing fast. Extract the loop into run_router_canary() and run it under wait_or_die so a dead prefill pid aborts it in ~5s like the surrounding barriers. Co-authored-by: Cursor --- .../multi_node/amd_utils/server_sglang.sh | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 5584bd87c3..f801bfa837 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -859,29 +859,37 @@ if [ "$NODE_RANK" -eq 0 ]; then # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed. # Gate the benchmark on ONE successful generation THROUGH the router so the eval # never starts against a router whose prefill path is not yet actually serving. - if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then - CANARY_URL="http://${NODE0_ADDR}:30000/v1/chat/completions" - CANARY_MODEL="${MODEL_DIR}/${MODEL_NAME}" - canary_ok=0 - canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) + # + # Run the poll loop under wait_or_die (like the barriers above) rather than + # inline: the canary is the FIRST real generation through prefill, so a + # prefill crash right after /readiness passes is plausible. Without + # wait_or_die watching prefill0_pid, that just looks like repeated 503s and + # the loop burns the full ROUTER_CANARY_TIMEOUT (default 600s) instead of + # detecting the dead pid and aborting in ~5s. + run_router_canary() { + local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions" + local canary_model="${MODEL_DIR}/${MODEL_NAME}" + local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) + local canary_code while [ "$(date +%s)" -lt "$canary_deadline" ]; do canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \ - -X POST "$CANARY_URL" -H 'Content-Type: application/json' \ - -d "{\"model\":\"${CANARY_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) + -X POST "$canary_url" -H 'Content-Type: application/json' \ + -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) if [ "$canary_code" = "200" ] && \ ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then - canary_ok=1; break + echo "Router readiness canary passed (end-to-end generation OK)" + return 0 fi echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." sleep 5 done - if [ "$canary_ok" -ne 1 ]; then - echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." - head -c 800 /tmp/router_canary.out 2>/dev/null - exit 1 - fi - echo "Router readiness canary passed (end-to-end generation OK)" + echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." + head -c 800 /tmp/router_canary.out 2>/dev/null + return 1 + } + if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then + wait_or_die "$prefill0_pid" run_router_canary || exit 1 fi echo "Router is ready for benchmarking"