diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index a3b3436400..3d6d7b8c19 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -409,12 +409,12 @@ DeepSeek-V4-Pro-DI: max_running_requests: 1024 chunked_prefill_size: 131072 context_length: 9217 - max_total_tokens: 262144 + max_total_tokens: 2097152 no_dp: max_running_requests: 128 chunked_prefill_size: 131072 context_length: 9217 - max_total_tokens: 262144 + max_total_tokens: 2097152 decode: mem_fraction_static: 0.85 prefill_round_robin_balance: true diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 549f83fdd1..f801bfa837 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -165,6 +165,7 @@ no_dp = decode.get('no_dp', {}) # Decode DP config print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') @@ -172,6 +173,7 @@ print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') # Decode EP-only config (EP enabled but DP disabled) print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"') s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') @@ -179,6 +181,7 @@ print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') # Decode no-DP config print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"') print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') +print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') @@ -216,12 +219,23 @@ fi if [[ "$DECODE_ENABLE_DP" == "true" ]]; then decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) + decode_context_length=$DECODE_CONTEXT_LENGTH_DP elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY + decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY else decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP + decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP +fi +# In PD-disaggregation the decode must admit requests against the SAME context +# length as prefill; otherwise decode accepts over-length requests that prefill +# rejects, and those requests hang forever waiting for a KV transfer that never +# comes (the 8k1k conc-500 straggler). Fall back to the prefill value if the +# decode context_length is not set in the model config, so the two always agree. +if [[ -z "$decode_context_length" ]]; then + decode_context_length=$prefill_context_length fi # When both DP and EP are enabled, override max-running-requests and dispatch tokens @@ -267,6 +281,10 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance" fi +if [[ -n "$decode_context_length" ]]; then + DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}" +fi + if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" fi @@ -750,26 +768,37 @@ if [ "$NODE_RANK" -eq 0 ]; then fi echo "Congratulations!!! All prefill and decode servers are up . . ." - # Router resilience: a single prefill worker doing huge long-context prefills - # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a - # concurrent burst. With defaults the circuit breaker opens after 10 failures - # and short-circuits the whole worker, so the aiperf profiling burst sees - # "No available prefill workers (all circuits open or unhealthy)" and aborts - # with 100% errors. Disable the breaker and relax health-check sensitivity so - # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS. - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - - # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix - # prefix reuse across the agentic trace; round_robin decode keeps the single - # decode worker fed evenly. cache_threshold / balance_*_threshold tune the - # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any - # of these via env. - ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" - ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" - ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" - ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" - ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then + # Agentic router config (main): long-context prefills can look unhealthy to + # the default circuit breaker during a concurrent burst. Disable the breaker + # and relax health-check sensitivity so a busy-but-alive worker is not + # ejected. cache_aware prefill routing exploits HiCache/radix prefix reuse + # across the agentic trace; round_robin decode keeps the single decode worker + # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS. + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" + ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" + ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" + ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" + ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + else + # DI router config (8k1k branch, run 28696443568): with defaults the per-worker + # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial. + # In that run every request 503'd from request #1 ("all circuits open or + # unhealthy") for ~31s and lm_eval (max_retries=5) then gave up -- i.e. the + # circuit was still open when the client budget ran out, so 0 result files were + # produced. Shortening the open->half-open window (and letting the router itself + # retry a failed worker selection) lets a transient trip re-close INSIDE the + # client retry budget instead of nuking the whole eval. The breaker stays fully + # ENABLED (thresholds unchanged); this only speeds recovery. Override via + # ROUTER_CB_ARGS / ROUTER_POLICY_FLAGS. + ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy random --prefill-policy random --decode-policy random}" + ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:-${ROUTER_CB_ARGS}}" + fi + + echo "Router config: IS_AGENTIC=${IS_AGENTIC:-0} policy/resilience=${ROUTER_POLICY_FLAGS} ${ROUTER_RESILIENCE_FLAGS}" ROUTER_CMD="python -m sglang_router.launch_router \ --pd-disaggregation \ @@ -822,6 +851,47 @@ if [ "$NODE_RANK" -eq 0 ]; then wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1 fi + # ---- End-to-end router readiness canary (run 28696443568) ---- + # The /readiness barrier above only proves the router PROCESS is up; it does + # NOT prove the router can reach a prefill worker and complete a generation. + # In that run the eval fired the instant /readiness passed and EVERY request + # 503'd ("No available prefill workers (all circuits open or unhealthy)") from + # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed. + # Gate the benchmark on ONE successful generation THROUGH the router so the eval + # never starts against a router whose prefill path is not yet actually serving. + # + # Run the poll loop under wait_or_die (like the barriers above) rather than + # inline: the canary is the FIRST real generation through prefill, so a + # prefill crash right after /readiness passes is plausible. Without + # wait_or_die watching prefill0_pid, that just looks like repeated 503s and + # the loop burns the full ROUTER_CANARY_TIMEOUT (default 600s) instead of + # detecting the dead pid and aborting in ~5s. + run_router_canary() { + local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions" + local canary_model="${MODEL_DIR}/${MODEL_NAME}" + local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} )) + local canary_code + while [ "$(date +%s)" -lt "$canary_deadline" ]; do + canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ + -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \ + -X POST "$canary_url" -H 'Content-Type: application/json' \ + -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) + if [ "$canary_code" = "200" ] && \ + ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then + echo "Router readiness canary passed (end-to-end generation OK)" + return 0 + fi + echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." + sleep 5 + done + echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." + head -c 800 /tmp/router_canary.out 2>/dev/null + return 1 + } + if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then + wait_or_die "$prefill0_pid" run_router_canary || exit 1 + fi + echo "Router is ready for benchmarking" fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ac4e276906..83e45f3008 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1166,17 +1166,34 @@ dsv4-fp4-mi355x-sglang-disagg: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - spec-decoding: "none" - conc-list: [ 128, 256 ] + conc-list: [ 256 ] prefill: - num-worker: 2 + num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + + - spec-decoding: "none" + conc-list: [ 512 ] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e82277b6f0..7b090359c3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4983,6 +4983,16 @@ - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281 +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg + description: + - "Retune the non-MTP 8k1k disagg search space: drop the 2P1D DEP8 sweep (conc 128/256/512/1024 across 2 prefill nodes) and the 1P1D DEP8 conc=1024 point" + - "Add 1P1D DEP8 (TP8/EP8 + dp-attention, 1 prefill node) at conc 256/512, and a new 1P1D pure-TP8 (no EP, no DP-attention) scenario sweeping conc [1, 2, 4, 8, 16, 32, 64, 128]" + - "Fix 2P1D eval 503s: speed up router circuit-breaker recovery (--cb-timeout-duration-secs 15 --retry-max-retries 3) and gate benchmarking on an end-to-end router readiness canary (one real generation through the router), so the eval no longer starts against a router whose prefill path isn't actually serving yet" + - "Fix the 8k1k conc-500 straggler tail: decode now derives its admission context_length from models.yaml (falling back to prefill's value when unset) so decode and prefill reject over-length requests symmetrically instead of decode hanging on a KV transfer prefill never sends" + - "Bump DeepSeek-V4-Pro prefill max_total_tokens from 262144 to 2097152 to match the model's actual KV budget" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2293 + - config-keys: - dsv4-fp4-b300-vllm-agentic scenario-type: