Larryli2/kimik2.5 fp4 eagle3 fp8

.github/configs/amd-master.yaml

@@ -546,6 +546,27 @@ kimik2.5-int4-mi355x-vllm:
       osl: 1024
       search-space:
       - { tp: 8, conc-start: 4, conc-end: 64 }
+
+kimik2.5-fp4-mi355x-vllm-eagle3-fp8:
+  image: vllm/vllm-openai-rocm:nightly-fb1ac806c55a6dc96fe92261b80c8550e9c39d2f
+  model: amd/Kimi-K2.5-MXFP4
+  model-prefix: kimik2.5
+  runner: mi355x
+  precision: fp4
+  framework: vllm
+  multinode: false
+  scenarios:
+    fixed-seq-len:
+    - isl: 1024
+      osl: 1024
+      search-space:
+      - { tp: 8, spec-decoding: eagle3_fp8, conc-start: 4, conc-end: 64 }
+      - { tp: 4, spec-decoding: eagle3_fp8, conc-start: 4, conc-end: 64 }
+    - isl: 8192
+      osl: 1024
+      search-space:
+      - { tp: 8, spec-decoding: eagle3_fp8, conc-start: 4, conc-end: 64 }
+      - { tp: 4, spec-decoding: eagle3_fp8, conc-start: 4, conc-end: 64 }
 
 kimik2.5-int4-mi325x-vllm:
   image: vllm/vllm-openai-rocm:v0.21.0

.github/workflows/README.md

@@ -271,7 +271,7 @@ The corresponding `SingleNodeMatrixEntry` enforces these same fields with approp
 
 2. **`extra='forbid'`**: Unknown fields are rejected, preventing typos or deprecated fields from slipping through.
 
-3. **Strict typing**: Fields like `spec-decoding` use `Literal["mtp", "draft_model", "none"]` to restrict values to known options.
+3. **Strict typing**: Fields like `spec-decoding` use a `Literal` type to restrict values to a fixed set of known options (see `utils/matrix_logic/validation.py` for the current set).
 
 4. **Concurrency validation**: The system ensures either `conc-list` OR `conc-start`/`conc-end` is provided, but not both.
 

benchmarks/single_node/kimik2.5_fp4_mi355x_eagle3_fp8.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    MODEL \
+    TP \
+    CONC \
+    ISL \
+    OSL \
+    MAX_MODEL_LEN \
+    RANDOM_RANGE_RATIO \
+    RESULT_FILENAME
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi
+
+# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+
+if [ -n "$ROCR_VISIBLE_DEVICES" ]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
+
+SERVER_LOG=/workspace/server.log
+PORT=${PORT:-8888}
+
+if [ "${EVAL_ONLY}" = "true" ]; then
+    setup_eval_context
+    MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
+fi
+
+# If the machine runs a MEC FW older than 177, RCCL
+# cannot reclaim some memory.
+# Disable that features to avoid crashes.
+# This is related to the changes in the driver at:
+# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates
+version=`rocm-smi --showfw | grep MEC | head -n 1 |  awk '{print $NF}'`
+if [[ "$version" == "" || $version -lt 177 ]]; then
+  export HSA_NO_SCRATCH_RECLAIM=1
+fi
+
+export VLLM_ROCM_USE_AITER=1
+# FP8 Eagle3 quantizes the draft model; the target model remains MXFP4.
+export VLLM_ROCM_USE_AITER_FP4_ASM_GEMM=1
+export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
+
+# Disable AITER RMSNorm for TP < 8 due to accuracy issues
+if [ "${TP}" -lt 8 ]; then
+  export VLLM_ROCM_USE_AITER_RMSNORM=0
+fi
+
+if [ "${EP_SIZE:-0}" -gt 1 ]; then
+  EP=" --enable-expert-parallel"
+else
+  EP=" "
+fi
+
+SPEC_DRAFT_MODEL=${SPEC_DRAFT_MODEL:-amd/kimi-k2.5-eagle3-fp8}
+SPEC_NUM_TOKENS=${SPEC_NUM_TOKENS:-6}
+SPEC_DRAFT_TP=${SPEC_DRAFT_TP:-1}
+if [[ "$SPEC_DRAFT_MODEL" != /* ]]; then hf download "$SPEC_DRAFT_MODEL"; fi
+SPEC_CONFIG="{\"model\":\"${SPEC_DRAFT_MODEL}\",\"method\":\"eagle3\",\"num_speculative_tokens\":${SPEC_NUM_TOKENS},\"draft_tensor_parallel_size\":${SPEC_DRAFT_TP}}"
+
+PREFILL_ARGS=()
+if [[ "${ISL}" -ge 8192 ]]; then
+  PREFILL_ARGS+=(
+    --long-prefill-token-threshold 8192
+    --max-num-batched-tokens 16384
+  )
+fi
+
+# Start GPU monitoring (power, temperature, clocks every second)
+start_gpu_monitor
+
+set -x
+vllm serve $MODEL --port $PORT \
+--tensor-parallel-size=$TP \
+$EP \
+--gpu-memory-utilization 0.90 \
+--max-model-len $MAX_MODEL_LEN \
+"${PREFILL_ARGS[@]}" \
+--no-enable-prefix-caching \
+--trust-remote-code \
+--mm-encoder-tp-mode data \
+--speculative-config "$SPEC_CONFIG" > $SERVER_LOG 2>&1 &
+
+SERVER_PID=$!
+
+# Wait for server to be ready
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+run_benchmark_serving \
+    --model "$MODEL" \
+    --port "$PORT" \
+    --backend vllm \
+    --input-len "$ISL" \
+    --output-len "$OSL" \
+    --random-range-ratio "$RANDOM_RANGE_RATIO" \
+    --num-prompts "$((CONC * 10))" \
+    --max-concurrency "$CONC" \
+    --result-filename "$RESULT_FILENAME" \
+    --result-dir /workspace/ \
+    --use-chat-template \
+    --trust-remote-code
+
+# After throughput, run evaluation only if RUN_EVAL is true
+if [ "${RUN_EVAL}" = "true" ]; then
+    run_eval --framework lm-eval --port "$PORT"
+    append_lm_eval_summary
+fi
+
+# Stop GPU monitoring
+stop_gpu_monitor
+set +x

perf-changelog.yaml

@@ -2901,3 +2901,12 @@
   description:
     - "Add MI355X config: qwen3.5-fp4-sglang-mtp using lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1445
+
+- config-keys:
+    - kimik2.5-fp4-mi355x-vllm-eagle3-fp8
+  description:
+    - "Add Kimi-K2.5 MXFP4 vLLM Eagle3 speculative decoding config with AMD Quark FP8 draft model on MI355X"
+    - "Image: vllm/vllm-openai-rocm:nightly-fb1ac806c55a6dc96fe92261b80c8550e9c39d2f"
+    - "Model: amd/Kimi-K2.5-MXFP4"
+    - "Draft model: amd/kimi-k2.5-eagle3-fp8 (num_speculative_tokens=6, draft_tp=1)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1515

runners/launch_mi355x-amds.sh

@@ -180,7 +180,11 @@ else
     export PORT_OFFSET=${RUNNER_NAME: -1}
     export PORT=$(( 8888 + ${PORT_OFFSET} ))
     FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '')
-    SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '')
+    case "$SPEC_DECODING" in
+        mtp)        SPEC_SUFFIX='_mtp' ;;
+        eagle3_fp8) SPEC_SUFFIX='_eagle3_fp8' ;;
+        *)          SPEC_SUFFIX='' ;;
+    esac
 
     PARTITION="compute"
     SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"

utils/matrix_logic/test_validation.py

@@ -281,7 +281,7 @@ def test_conc_as_list(self, valid_single_node_matrix_entry):
 
     def test_spec_decoding_values(self, valid_single_node_matrix_entry):
         """Spec decoding should accept valid literal values."""
-        for value in ["mtp", "draft_model", "none"]:
+        for value in ["mtp", "draft_model", "eagle3_fp8", "none"]:
             valid_single_node_matrix_entry["spec-decoding"] = value
             entry = SingleNodeMatrixEntry(**valid_single_node_matrix_entry)
             assert entry.spec_decoding == value

utils/matrix_logic/validation.py

@@ -86,7 +86,7 @@ class SingleNodeMatrixEntry(BaseModel):
     model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value)
     precision: str
     framework: str
-    spec_decoding: Literal["mtp", "draft_model", "none"] = Field(
+    spec_decoding: Literal["mtp", "draft_model", "eagle3_fp8", "none"] = Field(
         alias=Fields.SPEC_DECODING.value
     )
     runner: str
@@ -125,7 +125,7 @@ class MultiNodeMatrixEntry(BaseModel):
     model_prefix: str = Field(alias=Fields.MODEL_PREFIX.value)
     precision: str
     framework: str
-    spec_decoding: Literal["mtp", "draft_model", "none"] = Field(
+    spec_decoding: Literal["mtp", "draft_model", "eagle3_fp8", "none"] = Field(
         alias=Fields.SPEC_DECODING.value
     )
     runner: str
@@ -271,7 +271,7 @@ class SingleNodeSearchSpaceEntry(BaseModel):
 
     tp: int
     ep: Optional[int] = None
-    spec_decoding: Literal["mtp", "draft_model", "none"] = Field(
+    spec_decoding: Literal["mtp", "draft_model", "eagle3_fp8", "none"] = Field(
         default="none", alias=Fields.SPEC_DECODING.value)
     dp_attn: Optional[bool] = Field(
         default=None, alias=Fields.DP_ATTN.value)
@@ -291,7 +291,7 @@ class MultiNodeSearchSpaceEntry(BaseModel):
     """Multinode search space configuration."""
     model_config = ConfigDict(extra='forbid', populate_by_name=True)
 
-    spec_decoding: Literal["mtp", "draft_model", "none"] = Field(
+    spec_decoding: Literal["mtp", "draft_model", "eagle3_fp8", "none"] = Field(
         default="none", alias=Fields.SPEC_DECODING.value)
     prefill: WorkerConfig
     decode: WorkerConfig