Skip to content

Commit 009cff5

Browse files
seungrokjseungrokj
authored
Merge branch 'main' into amd/m3_atom_pd_fp4_0623
2 parents 4ff0e52 + 4f80171 commit 009cff5

7 files changed

Lines changed: 278 additions & 3 deletions

File tree

‎.github/configs/amd-master.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2722,8 +2722,27 @@ minimaxm3-fp4-mi355x-atom:
27222722
- { tp: 4, conc-start: 1, conc-end: 256 }
27232723
- { tp: 8, conc-start: 1, conc-end: 2 }
27242724

2725+
minimaxm3-fp8-mi355x-atom:
2726+
image: rocm/atom-dev:MiniMax-M3-20260623
2727+
model: MiniMaxAI/MiniMax-M3-MXFP8
2728+
model-prefix: minimaxm3
2729+
runner: mi355x
2730+
precision: fp8
2731+
framework: atom
2732+
multinode: false
2733+
scenarios:
2734+
fixed-seq-len:
2735+
- isl: 1024
2736+
osl: 1024
2737+
search-space:
2738+
- { tp: 4, conc-start: 1, conc-end: 256 }
2739+
- isl: 8192
2740+
osl: 1024
2741+
search-space:
2742+
- { tp: 4, conc-start: 1, conc-end: 256 }
2743+
27252744
minimaxm3-fp8-mi355x-atom-mtp:
2726-
image: rocm/atom-dev:MiniMax-M3-20260622
2745+
image: rocm/atom-dev:MiniMax-M3-20260623
27272746
model: MiniMaxAI/MiniMax-M3-MXFP8
27282747
model-prefix: minimaxm3
27292748
runner: mi355x

‎.github/configs/nvidia-master.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12375,6 +12375,43 @@ minimaxm3-fp8-b300-vllm:
1237512375
- { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128 }
1237612376
- { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 }
1237712377

12378+
# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4
12379+
# (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the
12380+
# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). MiniMax-M3
12381+
# modelopt NVFP4 support (vllm-project/vllm PR #46380) is baked into the perf
12382+
# container image, so no runtime patch is needed; prompts are routed through the
12383+
# chat template. Target weights are pre-staged read-only at
12384+
# /scratch/models/MiniMax-M3-NVFP4 (added to the STAGED_MODELS allow-list in
12385+
# launch_b300-nv.sh); the EAGLE3 draft is downloaded to the writable models dir.
12386+
minimaxm3-fp4-b300-vllm-mtp:
12387+
image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41
12388+
model: nvidia/MiniMax-M3-NVFP4
12389+
model-prefix: minimaxm3
12390+
runner: b300
12391+
precision: fp4
12392+
framework: vllm
12393+
multinode: false
12394+
scenarios:
12395+
fixed-seq-len:
12396+
- isl: 1024
12397+
osl: 1024
12398+
search-space:
12399+
- { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp }
12400+
- { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp }
12401+
- { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp }
12402+
- { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp }
12403+
- { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512, spec-decoding: mtp }
12404+
- { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp }
12405+
- isl: 8192
12406+
osl: 1024
12407+
search-space:
12408+
- { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp }
12409+
- { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp }
12410+
- { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp }
12411+
- { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp }
12412+
- { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp }
12413+
- { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp }
12414+
1237812415
# MiniMax-M3 day-zero (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3).
1237912416
# 427B total / 26B active MoE with MSA sparse attention; MXFP8 checkpoint
1238012417
# (MiniMaxAI/MiniMax-M3-MXFP8, ~444 GB) quantized by NVIDIA — native MX tensor

‎benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/usr/bin/env bash
2+
3+
# MiniMax-M3 NVFP4 B300 single-node vLLM recipe with EAGLE3 speculative
4+
# decoding — same shape as minimaxm3_fp8_b300_mtp.sh but uses the
5+
# nvidia/MiniMax-M3-NVFP4 checkpoint. MiniMax-M3 modelopt NVFP4 support
6+
# (vllm-project/vllm PR #46380) is baked into the perf container image, so no
7+
# runtime patch is needed.
8+
9+
source "$(dirname "$0")/../../benchmark_lib.sh"
10+
11+
check_env_vars \
12+
MODEL \
13+
TP \
14+
EP_SIZE \
15+
DP_ATTENTION \
16+
CONC \
17+
ISL \
18+
OSL \
19+
MAX_MODEL_LEN \
20+
RANDOM_RANGE_RATIO \
21+
RESULT_FILENAME
22+
23+
DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3"
24+
25+
# The target weights are launched from MODEL_PATH (the b300 launcher points it
26+
# at the pre-staged read-only /scratch/models/MiniMax-M3-NVFP4). The EAGLE3
27+
# draft is not pre-staged and must be downloaded, so it cannot live next to the
28+
# read-only target — fetch it into the writable models dir (/data/models)
29+
# instead. When MODEL_PATH is unset (stand-alone runs) fall back to the HF cache.
30+
if [[ -n "${MODEL_PATH:-}" ]]; then
31+
if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
32+
hf download "$MODEL" --local-dir "$MODEL_PATH"
33+
fi
34+
DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}"
35+
if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then
36+
hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH"
37+
fi
38+
else
39+
hf download "$MODEL"
40+
export MODEL_PATH="$MODEL"
41+
hf download "$DRAFT_MODEL"
42+
DRAFT_MODEL_PATH="$DRAFT_MODEL"
43+
fi
44+
45+
if [[ -n "$SLURM_JOB_ID" ]]; then
46+
echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
47+
fi
48+
49+
nvidia-smi
50+
51+
SERVER_LOG=/workspace/server.log
52+
53+
export VLLM_ENGINE_READY_TIMEOUT_S=3600
54+
export VLLM_FLOAT32_MATMUL_PRECISION=high
55+
56+
if [ "${DP_ATTENTION}" = "true" ]; then
57+
PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel"
58+
elif [ "$EP_SIZE" -gt 1 ]; then
59+
PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel"
60+
else
61+
PARALLEL_ARGS="--tensor-parallel-size=$TP"
62+
fi
63+
64+
# use 3 speculative tokens for all configs for now
65+
NUM_SPEC_TOKENS=3
66+
67+
if [ "${EVAL_ONLY}" = "true" ]; then
68+
setup_eval_context
69+
MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
70+
fi
71+
start_gpu_monitor
72+
73+
set -x
74+
vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \
75+
$PARALLEL_ARGS \
76+
--gpu-memory-utilization 0.90 \
77+
--max-model-len $MAX_MODEL_LEN \
78+
--block-size 128 \
79+
--language-model-only \
80+
--max-cudagraph-capture-size 2048 \
81+
--max-num-batched-tokens "$((ISL * 2 ))" \
82+
--speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \
83+
--stream-interval 20 --no-enable-prefix-caching \
84+
--trust-remote-code > $SERVER_LOG 2>&1 &
85+
86+
SERVER_PID=$!
87+
88+
wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
89+
90+
pip install -q datasets pandas
91+
92+
run_benchmark_serving \
93+
--model "$MODEL" \
94+
--port "$PORT" \
95+
--backend vllm \
96+
--input-len "$ISL" \
97+
--output-len "$OSL" \
98+
--random-range-ratio "$RANDOM_RANGE_RATIO" \
99+
--num-prompts "$((CONC * 10))" \
100+
--max-concurrency "$CONC" \
101+
--result-filename "$RESULT_FILENAME" \
102+
--result-dir /workspace/ \
103+
--trust-remote-code \
104+
--use-chat-template
105+
106+
if [ "${RUN_EVAL}" = "true" ]; then
107+
run_eval --framework lm-eval --port "$PORT"
108+
append_lm_eval_summary
109+
fi
110+
111+
stop_gpu_monitor
112+
set +x

‎benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom.sh‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/usr/bin/env bash
2+
3+
source "$(dirname "$0")/../../benchmark_lib.sh"
4+
5+
check_env_vars \
6+
MODEL \
7+
TP \
8+
CONC \
9+
ISL \
10+
OSL \
11+
RANDOM_RANGE_RATIO \
12+
RESULT_FILENAME \
13+
EP_SIZE \
14+
DP_ATTENTION
15+
16+
if [[ -n "$SLURM_JOB_ID" ]]; then
17+
echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
18+
fi
19+
20+
echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION"
21+
22+
SERVER_LOG=/workspace/server.log
23+
24+
PARALLEL_ARGS=(-tp "$TP") #TP
25+
if [ "$DP_ATTENTION" = "true" ]; then
26+
if [ "$EP_SIZE" -gt 1 ]; then #DP+EP
27+
PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention )
28+
else #DP+TP
29+
PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention )
30+
fi
31+
fi
32+
33+
SPEC_ARGS=()
34+
35+
# Start GPU monitoring (power, temperature, clocks every second)
36+
start_gpu_monitor
37+
MEM_FRAC_STATIC=0.8
38+
39+
set -x
40+
export AITER_QUICK_REDUCE_QUANTIZATION=INT4
41+
export AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0
42+
export ATOM_M3_SPARSE_USE_ASM_PA=1
43+
export MAX_MODEL_LEN=32768
44+
export MAX_NUM_BATCHED_TOKENS=32768
45+
export MAX_NUM_SEQS=256
46+
python3 -m atom.entrypoints.openai_server \
47+
--model $MODEL \
48+
--server-port $PORT \
49+
"${PARALLEL_ARGS[@]}" \
50+
"${SPEC_ARGS[@]}" \
51+
--block-size 128 \
52+
--gpu-memory-utilization $MEM_FRAC_STATIC \
53+
--max-model-len $MAX_MODEL_LEN \
54+
--max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \
55+
--max-num-seqs $MAX_NUM_SEQS \
56+
--kv_cache_dtype fp8 \
57+
--trust-remote-code \
58+
--no-enable_prefix_caching \
59+
> $SERVER_LOG 2>&1 &
60+
61+
SERVER_PID=$!
62+
63+
# Wait for server to be ready
64+
wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
65+
66+
export PYTHONDONTWRITEBYTECODE=1
67+
run_benchmark_serving \
68+
--model "$MODEL" \
69+
--port "$PORT" \
70+
--backend vllm \
71+
--input-len "$ISL" \
72+
--output-len "$OSL" \
73+
--random-range-ratio "$RANDOM_RANGE_RATIO" \
74+
--num-prompts "$((CONC * 10))" \
75+
--max-concurrency "$CONC" \
76+
--result-filename "$RESULT_FILENAME" \
77+
--result-dir /workspace/ \
78+
--trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" )
79+
80+
# After throughput, run evaluation only if RUN_EVAL is true
81+
if [ "${RUN_EVAL}" = "true" ]; then
82+
run_eval --framework lm-eval --port "$PORT"
83+
append_lm_eval_summary
84+
fi
85+
86+
# Stop GPU monitoring
87+
stop_gpu_monitor
88+
set +x

‎benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom_mtp.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ MEM_FRAC_STATIC=0.8
3838

3939
set -x
4040
export AITER_QUICK_REDUCE_QUANTIZATION=INT4
41+
export AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0
42+
export ATOM_M3_SPARSE_USE_ASM_PA=1
4143
export MAX_MODEL_LEN=32768
4244
export MAX_NUM_BATCHED_TOKENS=32768
4345
export MAX_NUM_SEQS=256
44-
# (srok), not yet
45-
# --kv_cache_dtype fp8 \
4646
python3 -m atom.entrypoints.openai_server \
4747
--model $MODEL \
4848
--server-port $PORT \
@@ -53,6 +53,7 @@ python3 -m atom.entrypoints.openai_server \
5353
--max-model-len $MAX_MODEL_LEN \
5454
--max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \
5555
--max-num-seqs $MAX_NUM_SEQS \
56+
--kv_cache_dtype fp8 \
5657
--trust-remote-code \
5758
--no-enable_prefix_caching \
5859
> $SERVER_LOG 2>&1 &

‎perf-changelog.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4001,6 +4001,14 @@
40014001
- "Remove the runtime SupportsEagle3 source patch now included in the pinned nightly"
40024002
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1843
40034003

4004+
- config-keys:
4005+
- minimaxm3-fp8-mi355x-atom
4006+
- minimaxm3-fp8-mi355x-atom-mtp
4007+
description:
4008+
- "Add minimaxm3-fp8-mi355x-atom: MiniMax-M3 MXFP8 single-node benchmark on MI355X using ATOM framework"
4009+
- "Uses rocm/atom-dev:MiniMax-M3-20260623; TP4, block size 128, ISL=1024,8192 OSL=1024"
4010+
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1916
4011+
40044012
- config-keys:
40054013
- minimaxm3-fp8-mi355x-atom-mtp
40064014
description:
@@ -4197,6 +4205,15 @@
41974205
- "Initial submission: MiniMax-M3 MXFP4 disagg (prefill/decode) on MI355X with vLLM over the MoRI-IO KV connector (8k/1k)."
41984206
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1914
41994207

4208+
- config-keys:
4209+
- minimaxm3-fp4-b300-vllm-mtp
4210+
description:
4211+
- "Add MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node aggregated vLLM benchmark with EAGLE3 speculative decoding (spec-decoding: mtp, 3 draft tokens via Inferact/MiniMax-M3-EAGLE3)"
4212+
- "Image vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 (bakes in MiniMax-M3 modelopt NVFP4 support, vllm-project/vllm PR #46380; no runtime patch needed); prompts routed through the chat template"
4213+
- "Target weights pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added MiniMax-M3-NVFP4 to launch_b300-nv.sh STAGED_MODELS); EAGLE3 draft downloaded to the writable /data/models; --block-size 128 (MSA), --language-model-only"
4214+
- "Sweeps tp 4/8 with and without EP and dp-attn at 1k1k and 8k1k, conc 1-512"
4215+
pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1929
4216+
42004217
- config-keys:
42014218
- minimaxm3-fp4-mi355x-vllm
42024219
description:

‎runners/launch_b300-nv.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,7 @@ else
362362
MiniMax-M2.7
363363
MiniMax-M2.7-NVFP4
364364
MiniMax-M3
365+
MiniMax-M3-NVFP4
365366
Qwen3.5-397B-A17B
366367
Qwen3.5-397B-A17B-FP8
367368
Qwen3.5-397B-A17B-NVFP4

0 commit comments

Comments
 (0)