Merge branch 'main' into dsv4-fp4-gb300-1k1k-stp-disagg

Author: hshrivastava-droid

benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh

@@ -3,17 +3,12 @@
 # MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe with EAGLE3
 # speculative decoding — the spec-decoding=mtp variant of
 # minimaxm3_fp8_mi355x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via
-# --speculative-config with 3 speculative tokens. Everything else mirrors the
-# non-MTP recipe: MXFP8 from TP=4 on gfx950, mandatory --block-size 128,
-# --language-model-only for the text-only benchmark, FP8 KV cache, and
-# --attention-backend TRITON_ATTN. Runs with CUDA graphs (no --enforce-eager);
-# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path.
+# --speculative-config with 3 speculative tokens. 
 #
-# Unlike the CUDA recipes, the drafter needs no attention_backend override:
-# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced
-# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific.
-# Here the whole server runs on TRITON_ATTN (set globally below), which serves
-# the MHA draft fine.
+# The EAGLE3 drafter (dense Llama MHA head) is pinned to TRITON_ATTN in the
+# speculative-config, otherwise it would fall back to a slow default backend.
+# Adding the explicit override left the draft's token acceptance unchanged but
+# sped up the draft forward enough to turn into a win across the board.
 #
 # [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image
 # does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3
@@ -197,7 +192,7 @@ vllm serve "$MODEL" --port "$PORT" \
     --kv-cache-dtype fp8 \
     --attention-backend TRITON_ATTN \
     --linear-backend emulation \
-    --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \
+    --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \
     --tool-call-parser minimax_m3 \
     --reasoning-parser minimax_m3 \
     --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 &

perf-changelog.yaml

@@ -4575,6 +4575,13 @@
     - "Add high concurrency configs"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994
 
+- config-keys:
+    - minimaxm3-fp8-mi355x-vllm-mtp
+  description:
+    - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend."
+    - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107
+
 - config-keys:
     - dsv4-fp4-gb300-dynamo-sglang
   description: