|
| 1 | +#!/usr/bin/env bash |
| 2 | +set -euo pipefail |
| 3 | +set -x |
| 4 | + |
| 5 | +# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. |
| 6 | +# |
| 7 | +# Required env vars: |
| 8 | +# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, |
| 9 | +# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION |
| 10 | + |
| 11 | +source "$(dirname "$0")/../../benchmark_lib.sh" |
| 12 | + |
| 13 | +# Force the eval framework to lm-eval for this recipe. run_eval derives its |
| 14 | +# default as swebench for agentic scenarios (scenario_default=swebench when |
| 15 | +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence |
| 16 | +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so |
| 17 | +# setting it here makes the effective framework always lm-eval, never swebench. |
| 18 | +export EVAL_FRAMEWORK="lm-eval" |
| 19 | + |
| 20 | +check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION |
| 21 | + |
| 22 | +echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" |
| 23 | + |
| 24 | +PORT=8888 |
| 25 | + |
| 26 | +if [[ -n "${SLURM_JOB_ID+x}" ]]; then |
| 27 | + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" |
| 28 | +fi |
| 29 | + |
| 30 | +# ROCR/HIP visibility for vLLM 0.14+ |
| 31 | +if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then |
| 32 | + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" |
| 33 | +fi |
| 34 | + |
| 35 | +if [[ -n "${MODEL_PATH:-}" ]]; then |
| 36 | + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then |
| 37 | + hf download "$MODEL" --local-dir "$MODEL_PATH" |
| 38 | + fi |
| 39 | +else |
| 40 | + hf download "$MODEL" |
| 41 | + export MODEL_PATH="$MODEL" |
| 42 | +fi |
| 43 | + |
| 44 | +rocm-smi || true |
| 45 | +amd-smi || true |
| 46 | + |
| 47 | +resolve_trace_source |
| 48 | +install_agentic_deps |
| 49 | + |
| 50 | +# ---- Server config ---------------------------------------------------------- |
| 51 | +SERVER_LOG="$RESULT_DIR/server.log" |
| 52 | +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" |
| 53 | +mkdir -p "$RESULT_DIR" |
| 54 | + |
| 55 | +OFFLOAD_ARGS=(--no-enable-prefix-caching) |
| 56 | + |
| 57 | +case "$KV_OFFLOAD_BACKEND" in |
| 58 | + vllm-simple) |
| 59 | + unset VLLM_USE_SIMPLE_KV_OFFLOAD |
| 60 | + # Use vLLM's regular native KV-offload path (OffloadingConnector), |
| 61 | + # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to |
| 62 | + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 |
| 63 | + # would switch it to SimpleCPUOffloadConnector. We intentionally leave |
| 64 | + # that env var UNSET here so the regular OffloadingConnector path is |
| 65 | + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size |
| 66 | + # form constructs the KVTransferConfig at engine startup |
| 67 | + # (vllm/config/vllm.py:662). |
| 68 | + |
| 69 | + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) |
| 70 | + # This gives extra cache hit than disabling hybrid kv cache manager |
| 71 | + OFFLOAD_ARGS=( |
| 72 | + --kv_offloading_backend native |
| 73 | + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" |
| 74 | + ) |
| 75 | + ;; |
| 76 | +esac |
| 77 | + |
| 78 | +# ---- LLM server config ---------------------------------------------------------- |
| 79 | +PARALLEL_ARGS=(--tensor-parallel-size "$TP") |
| 80 | +if [ "${DP_ATTENTION}" = "true" ]; then |
| 81 | + PARALLEL_ARGS=( |
| 82 | + --tensor-parallel-size 1 |
| 83 | + --data-parallel-size "$TP" |
| 84 | + --enable-expert-parallel |
| 85 | + ) |
| 86 | +elif [ "$EP_SIZE" -gt 1 ]; then |
| 87 | + PARALLEL_ARGS+=(--enable-expert-parallel) |
| 88 | +fi |
| 89 | + |
| 90 | +echo "Starting vllm server..." |
| 91 | +export PYTHONNOUSERSITE=1 |
| 92 | + |
| 93 | +export VLLM_ENGINE_READY_TIMEOUT_S=3600 |
| 94 | +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 |
| 95 | +export VLLM_ROCM_USE_AITER=1 |
| 96 | +export VLLM_ROCM_USE_AITER_MOE=1 |
| 97 | +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 |
| 98 | +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the |
| 99 | +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is |
| 100 | +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, |
| 101 | +# so it never fires for decode-sized tensors without it. |
| 102 | +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 |
| 103 | +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 |
| 104 | +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 |
| 105 | + |
| 106 | +VLLM_CMD=( |
| 107 | + vllm serve "$MODEL_PATH" |
| 108 | + --served-model-name "$MODEL" |
| 109 | + --host 0.0.0.0 |
| 110 | + --port "$PORT" |
| 111 | + "${PARALLEL_ARGS[@]}" |
| 112 | + --trust-remote-code |
| 113 | + --block-size 128 |
| 114 | + --gpu-memory-utilization 0.85 |
| 115 | + --language-model-only |
| 116 | + --attention-backend TRITON_ATTN |
| 117 | + --moe-backend aiter |
| 118 | + --kv-cache-dtype fp8 |
| 119 | + --tool-call-parser minimax_m3 |
| 120 | + --enable-auto-tool-choice |
| 121 | + # NOTE: --reasoning-parser minimax_m3 is intentionally OMITTED. |
| 122 | + # MiniMax-M3 is an interleaved-thinking model: its <mm:think>...</mm:think> |
| 123 | + # block MUST be round-tripped back into the conversation history every turn |
| 124 | + # or multi-turn quality collapses (the model loses its plan and degenerates |
| 125 | + # into repeating the same command until the step limit -> empty patch). |
| 126 | + # The reasoning parser moves <mm:think> out of message.content into the |
| 127 | + # response-only reasoning_content field, which the mini-swe-agent/litellm |
| 128 | + # OpenAI client does NOT resend. Leaving the parser off keeps the think block |
| 129 | + # inline in message.content, so the client preserves it across turns. The |
| 130 | + # tool-call parser above still extracts tool calls from the full output. |
| 131 | + --max-num-seqs "$CONC" |
| 132 | + "${OFFLOAD_ARGS[@]}" |
| 133 | +) |
| 134 | +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" |
| 135 | +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" |
| 136 | +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & |
| 137 | +SERVER_PID=$! |
| 138 | +echo "Server PID: $SERVER_PID" |
| 139 | + |
| 140 | +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" |
| 141 | + |
| 142 | +# ---- Run benchmark ---------------------------------------------------------- |
| 143 | +if [ "${EVAL_ONLY}" = "true" ]; then |
| 144 | + run_eval --port "$PORT" |
| 145 | +else |
| 146 | + build_replay_cmd "$RESULT_DIR" |
| 147 | + run_agentic_replay_and_write_outputs "$RESULT_DIR" |
| 148 | +fi |
0 commit comments