merge: sync upstream main

Merge current upstream/main and rebuild the append-only performance changelog with PR #2302 at the tail.

中文：同步上游 main。合并当前 upstream/main，并基于最新文件重新在性能变更日志末尾追加 PR #2302 条目。

Author: ivanium

benchmarks/multi_node/amd_utils/models.yaml

@@ -409,12 +409,12 @@ DeepSeek-V4-Pro-DI:
       max_running_requests: 1024
       chunked_prefill_size: 131072
       context_length: 9217
-      max_total_tokens: 262144
+      max_total_tokens: 2097152
     no_dp:
       max_running_requests: 128
       chunked_prefill_size: 131072
       context_length: 9217
-      max_total_tokens: 262144
+      max_total_tokens: 2097152
   decode:
     mem_fraction_static: 0.85
     prefill_round_robin_balance: true

benchmarks/multi_node/amd_utils/server_sglang.sh

@@ -165,20 +165,23 @@ no_dp = decode.get('no_dp', {})
 # Decode DP config
 print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{dp.get(\"max_running_requests\", 4096)}\"')
 print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"')
+print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"')
 s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160)
 print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"')
 print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"')
 
 # Decode EP-only config (EP enabled but DP disabled)
 print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"')
 print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"')
+print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"')
 s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256)
 print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"')
 print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"')
 
 # Decode no-DP config
 print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{no_dp.get(\"max_running_requests\", 128)}\"')
 print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"')
+print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"')
 s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128)
 print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"')
 print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"')
@@ -216,12 +219,23 @@ fi
 if [[ "$DECODE_ENABLE_DP" == "true" ]]; then
     decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END))
     decode_max_running_requests=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE))
+    decode_context_length=$DECODE_CONTEXT_LENGTH_DP
 elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then
     decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END))
     decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY
+    decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY
 else
     decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END))
     decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP
+    decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP
+fi
+# In PD-disaggregation the decode must admit requests against the SAME context
+# length as prefill; otherwise decode accepts over-length requests that prefill
+# rejects, and those requests hang forever waiting for a KV transfer that never
+# comes (the 8k1k conc-500 straggler). Fall back to the prefill value if the
+# decode context_length is not set in the model config, so the two always agree.
+if [[ -z "$decode_context_length" ]]; then
+    decode_context_length=$prefill_context_length
 fi
 
 # When both DP and EP are enabled, override max-running-requests and dispatch tokens
@@ -267,6 +281,10 @@ DECODE_MODE_FLAGS="--mem-fraction-static ${DECODE_MEM_FRACTION_STATIC} --max-run
 if [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "True" ]] || [[ "$DECODE_PREFILL_ROUND_ROBIN_BALANCE" == "true" ]]; then
     DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --prefill-round-robin-balance"
 fi
+if [[ -n "$decode_context_length" ]]; then
+    DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}"
+fi
+
 if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then
     DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache"
 fi
@@ -750,26 +768,37 @@ if [ "$NODE_RANK" -eq 0 ]; then
     fi
     echo "Congratulations!!! All prefill and decode servers are up . . ."
 
-    # Router resilience: a single prefill worker doing huge long-context prefills
-    # (256K+ token agentic prompts in 65280-token chunks) can be slow to drain a
-    # concurrent burst. With defaults the circuit breaker opens after 10 failures
-    # and short-circuits the whole worker, so the aiperf profiling burst sees
-    # "No available prefill workers (all circuits open or unhealthy)" and aborts
-    # with 100% errors. Disable the breaker and relax health-check sensitivity so
-    # a busy-but-alive worker is not ejected. Override via ROUTER_RESILIENCE_FLAGS.
-    ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}"
-
-    # Router scheduling policy. cache_aware prefill routing exploits HiCache/radix
-    # prefix reuse across the agentic trace; round_robin decode keeps the single
-    # decode worker fed evenly. cache_threshold / balance_*_threshold tune the
-    # cache_aware load-balancing (router defaults are 0.5 / 64 / 1.5). Override any
-    # of these via env.
-    ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}"
-    ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}"
-    ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}"
-    ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}"
-    ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}"
-    ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}"
+    if [[ "${IS_AGENTIC:-0}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then
+        # Agentic router config (main): long-context prefills can look unhealthy to
+        # the default circuit breaker during a concurrent burst. Disable the breaker
+        # and relax health-check sensitivity so a busy-but-alive worker is not
+        # ejected. cache_aware prefill routing exploits HiCache/radix prefix reuse
+        # across the agentic trace; round_robin decode keeps the single decode worker
+        # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS.
+        ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}"
+        ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}"
+        ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}"
+        ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}"
+        ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}"
+        ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}"
+        ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}"
+    else
+        # DI router config (8k1k branch, run 28696443568): with defaults the per-worker
+        # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial.
+        # In that run every request 503'd from request #1 ("all circuits open or
+        # unhealthy") for ~31s and lm_eval (max_retries=5) then gave up -- i.e. the
+        # circuit was still open when the client budget ran out, so 0 result files were
+        # produced. Shortening the open->half-open window (and letting the router itself
+        # retry a failed worker selection) lets a transient trip re-close INSIDE the
+        # client retry budget instead of nuking the whole eval. The breaker stays fully
+        # ENABLED (thresholds unchanged); this only speeds recovery. Override via
+        # ROUTER_CB_ARGS / ROUTER_POLICY_FLAGS.
+        ROUTER_CB_ARGS="${ROUTER_CB_ARGS:---cb-timeout-duration-secs 15 --retry-max-retries 3}"
+        ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy random --prefill-policy random --decode-policy random}"
+        ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:-${ROUTER_CB_ARGS}}"
+    fi
+
+    echo "Router config: IS_AGENTIC=${IS_AGENTIC:-0} policy/resilience=${ROUTER_POLICY_FLAGS} ${ROUTER_RESILIENCE_FLAGS}"
 
     ROUTER_CMD="python -m sglang_router.launch_router \
         --pd-disaggregation \
@@ -822,6 +851,47 @@ if [ "$NODE_RANK" -eq 0 ]; then
             wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1
         fi
 
+        # ---- End-to-end router readiness canary (run 28696443568) ----
+        # The /readiness barrier above only proves the router PROCESS is up; it does
+        # NOT prove the router can reach a prefill worker and complete a generation.
+        # In that run the eval fired the instant /readiness passed and EVERY request
+        # 503'd ("No available prefill workers (all circuits open or unhealthy)") from
+        # request #1 -> lm_eval gave up -> 0 result files -> "Verify eval scores" failed.
+        # Gate the benchmark on ONE successful generation THROUGH the router so the eval
+        # never starts against a router whose prefill path is not yet actually serving.
+        #
+        # Run the poll loop under wait_or_die (like the barriers above) rather than
+        # inline: the canary is the FIRST real generation through prefill, so a
+        # prefill crash right after /readiness passes is plausible. Without
+        # wait_or_die watching prefill0_pid, that just looks like repeated 503s and
+        # the loop burns the full ROUTER_CANARY_TIMEOUT (default 600s) instead of
+        # detecting the dead pid and aborting in ~5s.
+        run_router_canary() {
+            local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions"
+            local canary_model="${MODEL_DIR}/${MODEL_NAME}"
+            local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT:-600} ))
+            local canary_code
+            while [ "$(date +%s)" -lt "$canary_deadline" ]; do
+                canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \
+                    -m "${ROUTER_CANARY_REQ_TIMEOUT:-120}" \
+                    -X POST "$canary_url" -H 'Content-Type: application/json' \
+                    -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null)
+                if [ "$canary_code" = "200" ] && \
+                   ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then
+                    echo "Router readiness canary passed (end-to-end generation OK)"
+                    return 0
+                fi
+                echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ."
+                sleep 5
+            done
+            echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT:-600}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router."
+            head -c 800 /tmp/router_canary.out 2>/dev/null
+            return 1
+        }
+        if [[ "${ROUTER_READINESS_CANARY:-1}" == "1" ]]; then
+            wait_or_die "$prefill0_pid" run_router_canary || exit 1
+        fi
+
         echo "Router is ready for benchmarking"
     fi
 

benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh

@@ -3,6 +3,17 @@
 # MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe.
 # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5
 # The recipe recommends MXFP8 from TP=4 on gfx950 and requires block size 128.
+#
+# AITER page-16 sparse paged-attention fast path (vllm-project/vllm#47287,
+# merged into the pinned nightly): maps MiniMax-M3's top-k 128-token sparse
+# blocks onto AITER page-16 block tables and runs AITER Gluon paged attention
+# over only the selected KV pages. This is a kernel-level speedup of the same
+# sparse-attention computation (no FLOP reduction), enabled via
+# VLLM_ROCM_USE_AITER=1 + VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 with fp8 KV cache
+# on a TP where each rank has num_kv_heads == 1 (TP4). We deliberately do NOT
+# pass the #47269 --hf-overrides use_index_cache/index_topk_freq cross-layer
+# indexer-skip override: it reduces model-architecture FLOPs, which is
+# disallowed by docs/PR_REVIEW_CHECKLIST.md.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -31,10 +42,39 @@ fi
 SERVER_LOG=/workspace/server.log
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
 export VLLM_USE_BREAKABLE_CUDAGRAPH=0
-# MI355X mxfp8 recipe (vllm-project/recipes#581): INT6 quick all-reduce plus
-# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). 
+# MI355X mxfp8 recipe (vllm-project/recipes#581): INT4 quick all-reduce plus
+# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). INT4
+# quick all-reduce is applied at all concurrencies (accuracy is guarded by the
+# 8k1k evals); #2003 used INT6.
 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1
-export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6
+export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
+
+# AITER page-16 sparse PA (vllm-project/vllm#47287) is a long-context,
+# high-concurrency optimization: it maps MiniMax-M3's top-k 128-token sparse
+# blocks onto AITER page-16 block tables. Measured on gfx950 MXFP8, it only wins
+# in the 8k1k high-concurrency tail and adds overhead at short context (1k1k) or
+# low batch. So enable the "high-conc fast path" (shuffled KV-cache layout for
+# sparse PA + the emulation dense-linear backend, see below) only for
+# isl>=8192 && conc>=64; everywhere else fall back to the #2003 path
+# (non-shuffled Triton attention + native linear). Overridable via
+# MM3_HIGH_CONC_FASTPATH=0/1.
+if [ -z "${MM3_HIGH_CONC_FASTPATH:-}" ]; then
+    if [ "$ISL" -ge 8192 ] && [ "$CONC" -ge 64 ]; then
+        MM3_HIGH_CONC_FASTPATH=1
+    else
+        MM3_HIGH_CONC_FASTPATH=0
+    fi
+fi
+
+if [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then
+    export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1
+    # Quick all-reduce tuning from the MiniMax-M3 AITER recipe (vllm-project/vllm#47287):
+    # keep the bf16 accumulation and only quantize all-reduces above 256 KB.
+    export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0
+    export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256
+else
+    export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0
+fi
 
 if [ "${EVAL_ONLY}" = "true" ]; then
     setup_eval_context
@@ -61,6 +101,18 @@ export VLLM_ROCM_USE_AITER=1
 # concurrency. Overridable via env.
 MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}"
 
+# Dense-linear backend, gated on the same high-conc fast path as sparse PA. On
+# this nightly the native Triton MXFP8 linear GEMM wins in the memory-bound
+# low-concurrency regime, while --linear-backend emulation (bf16 hipBLASLT) wins
+# in the compute-bound high-concurrency regime (~+3-5% at 8k1k conc>=64).
+# LINEAR_BACKEND overrides (a backend name to force it, or "native" to disable).
+LINEAR_ARGS=()
+if [ -n "${LINEAR_BACKEND:-}" ]; then
+    [ "$LINEAR_BACKEND" != "native" ] && LINEAR_ARGS=(--linear-backend "$LINEAR_BACKEND")
+elif [ "$MM3_HIGH_CONC_FASTPATH" = "1" ]; then
+    LINEAR_ARGS=(--linear-backend emulation)
+fi
+
 start_gpu_monitor
 
 set -x
@@ -74,7 +126,7 @@ vllm serve "$MODEL" --port "$PORT" \
     --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \
     --kv-cache-dtype fp8 \
     --attention-backend TRITON_ATTN \
-    --linear-backend emulation \
+    "${LINEAR_ARGS[@]}" \
     --tool-call-parser minimax_m3 \
     --reasoning-parser minimax_m3 \
     --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 &

configs/amd-master.yaml

@@ -1166,17 +1166,34 @@ dsv4-fp4-mi355x-sglang-disagg:
           additional-settings:
           - "DECODE_NODES=1"
           - "DECODE_MTP_SIZE=0"
-
       # 1P1D DEP8  (mori KV transfer + mori MoE a2a, dp-attention)
       - spec-decoding: "none"
-        conc-list: [ 128, 256 ]
+        conc-list: [ 256 ]
         prefill:
-          num-worker: 2
+          num-worker: 1
           tp: 8
           ep: 8
           dp-attn: true
           additional-settings:
-          - "PREFILL_NODES=2"
+          - "PREFILL_NODES=1"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 8
+          dp-attn: true
+          additional-settings:
+          - "DECODE_NODES=1"
+          - "DECODE_MTP_SIZE=0"
+
+      - spec-decoding: "none"
+        conc-list: [ 512 ]
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 8
+          dp-attn: true
+          additional-settings:
+          - "PREFILL_NODES=1"
         decode:
           num-worker: 1
           tp: 8
@@ -1604,7 +1621,7 @@ dsv4-fp4-mi355x-atom-disagg:
           - "DECODE_NODES=1"
       # 1P1D TP8
 minimaxm3-fp8-mi355x-vllm:
-  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  image: vllm/vllm-openai-rocm:nightly-9e57de7197f234f9d9187715d96e07e007048c0f
   model: MiniMaxAI/MiniMax-M3-MXFP8
   model-prefix: minimaxm3
   runner: mi355x