From e80e44ac1341b46464adace6536247bfbd3a1ec3 Mon Sep 17 00:00:00 2001 From: haic0 Date: Mon, 20 Jul 2026 04:19:13 -0400 Subject: [PATCH 1/9] Update new MTP methods for kimik2.7_fp4_mi355x model for agentic CI workflow Signed-off-by: haic0 Co-authored-by: Cursor --- .../agentic/kimik2.7_fp4_mi355x_mtp.sh | 388 ++++++++++++++++++ configs/amd-master.yaml | 21 + perf-changelog.yaml | 8 + 3 files changed, 417 insertions(+) create mode 100755 benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh new file mode 100755 index 0000000000..b39e89a333 --- /dev/null +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM. +# KV_OFFLOADING=none -> GPU KV only +# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector +# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=mooncake -> Mooncake embedded store + connector +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE + + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +rocm-smi || true +amd-smi || true + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug) +pip install amd-quark + +# Disable AITER RMSNorm for TP < 8 due to accuracy issues +if [ "${TP}" -lt 8 ]; then + export VLLM_ROCM_USE_AITER_RMSNORM=0 +fi +# Workaround for MEC FW <177 RCCL memory reclaim issue +version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') +if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then + export HSA_NO_SCRATCH_RECLAIM=1 +fi + +export VLLM_ROCM_USE_AITER=1 +# This model has 64 KV heads. AMD flags AITER-MLA as officially supporting only +# 16/128 heads, but empirically it runs this model and (forced via --block-size=1 +# below) is the ONLY ROCm MLA backend that accepts block_size=1 and is ~3x faster +# than TRITON_MLA on the agentic trace (best-perf config, run 29337493492). +# Deliberately keeping AITER-MLA ON for throughput; revisit if accuracy validation +# shows the 64-head path is numerically wrong (then AITER_MLA=0 + drop block-size). +export VLLM_ROCM_USE_AITER_MLA=1 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# Avoid intermittent symm_mem all-reduce rendezvous hang at engine init on +# MI35x nodes (see KIMIK27_CONC64_LMCACHE_RUNBOOK error #4). +export VLLM_ALLREDUCE_USE_SYMM_MEM="${VLLM_ALLREDUCE_USE_SYMM_MEM:-0}" + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=() +PREFIX_CACHE_ARGS=() + +# ---- Offload service cleanup ------------------------------------------------ +LMCACHE_PID="" +MOONCAKE_MASTER_PID="" +ROUTER_PID="" + +cleanup_offload_services() { + if [[ -n "$ROUTER_PID" ]] && kill -0 "$ROUTER_PID" 2>/dev/null; then + kill "$ROUTER_PID" 2>/dev/null || true + wait "$ROUTER_PID" 2>/dev/null || true + fi + if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then + kill "$LMCACHE_PID" 2>/dev/null || true + wait "$LMCACHE_PID" 2>/dev/null || true + fi + if [[ -n "$MOONCAKE_MASTER_PID" ]] && kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true + wait "$MOONCAKE_MASTER_PID" 2>/dev/null || true + fi +} +trap cleanup_offload_services EXIT + +wait_for_lmcache_ready() { + { set +x; } 2>/dev/null + local attempts="${LMCACHE_READY_ATTEMPTS:-120}" + local tail_pid="" + + while [ ! -f "$LMCACHE_LOG" ]; do + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before creating log file. Exiting." >&2 + exit 1 + fi + sleep 1 + done + + tail -f -n +1 "$LMCACHE_LOG" & + tail_pid=$! + + for ((i = 1; i <= attempts; i++)); do + if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + return 0 + fi + if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "LMCache server died before becoming healthy. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 + fi + sleep 1 + done + + echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + cat "$LMCACHE_LOG" >&2 || true + exit 1 +} + +if [[ "$KV_OFFLOADING" == "none" ]]; then + OFFLOAD_MODE="none" +else + OFFLOAD_MODE="${KV_OFFLOAD_BACKEND:?KV_OFFLOAD_BACKEND required when KV_OFFLOADING=dram}" +fi + +case "$OFFLOAD_MODE" in + none) + # GPU-only KV baseline: keep on-GPU prefix caching ON (no DRAM offload) + # so it's apples-to-apples vs the lmcache cell. + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + ;; + lmcache) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + # Build LMCache against ROCm if the connector isn't importable. Clone to + # a container-local dir (NOT bind-mounted /workspace) so the next job's + # checkout `clean: true` won't trip over root-owned build artifacts. + if ! python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null 2>&1; then + LMCACHE_SRC_DIR="${LMCACHE_SRC_DIR:-/opt/lmcache-src}" + LMCACHE_GIT_REF="${LMCACHE_GIT_REF:-aaf7c0d3}" + rm -rf "$LMCACHE_SRC_DIR" + git clone https://github.com/LMCache/LMCache.git "$LMCACHE_SRC_DIR" + ( cd "$LMCACHE_SRC_DIR" + git checkout "$LMCACHE_GIT_REF" + pip install -r requirements/build.txt + CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation ) + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + fi + + LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" + LMCACHE_PORT="${LMCACHE_PORT:-5555}" + LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" + LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" + # LMCache L1 is SHM-backed: if L1 > /dev/shm free it silently disables + # SHM and falls back to the pickle path (crashes at load). Cap L1 to 90% + # of /dev/shm free so SHM stays enabled. + SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') + SHM_CAP_GB=$(( SHM_FREE_GB * 90 / 100 )) + LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" + if [ -n "$SHM_CAP_GB" ] && [ "$SHM_CAP_GB" -gt 0 ] && [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then + echo "Capping LMCACHE_L1_SIZE_GB $LMCACHE_L1_SIZE_GB -> $SHM_CAP_GB to fit /dev/shm (${SHM_FREE_GB}G free)" + LMCACHE_L1_SIZE_GB="$SHM_CAP_GB" + fi + LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" + LMCACHE_L1_READ_TTL_SECONDS="${LMCACHE_L1_READ_TTL_SECONDS:-7200}" + LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" + LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$((TP * 2))}" + export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" + export LMCACHE_BLOCKING_TIMEOUT_SECS=60 + + echo "Starting LMCache MP server..." + LMCACHE_CMD=( + lmcache server + --host "$LMCACHE_HOST" + --port "$LMCACHE_PORT" + --http-host "$LMCACHE_HOST" + --http-port "$LMCACHE_HTTP_PORT" + --l1-size-gb "$LMCACHE_L1_SIZE_GB" + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" + --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" + --chunk-size "$LMCACHE_CHUNK_SIZE" + --max-workers "$LMCACHE_MAX_WORKERS" + --eviction-policy LRU + ) + printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" + printf '\n' >> "$RESULT_DIR/lmcache_command.txt" + "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & + LMCACHE_PID=$! + echo "LMCache server PID: $LMCACHE_PID" + wait_for_lmcache_ready + + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" + ) + ;; + mooncake) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + # Mooncake embedded mode contributes one global segment per GPU rank to + # a shared distributed store, so pre-divide the aggregate host-memory + # budget across TP ranks. Mirrors the MI355X DSv4 vLLM recipe. + MOONCAKE_PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + + # No prebuilt ROCm wheel: build the Mooncake transfer engine from source + # if the store module isn't importable. Clone to a container-local dir + # (NOT bind-mounted /workspace) so the next job's checkout `clean: true` + # won't trip over root-owned build artifacts. + # + # A bare `make install` only lays down the C++ libs, so the launcher can + # import `mooncake` but the vLLM worker SUBPROCESSES cannot (fresh + # site-packages) -> "Please install mooncake ..." at KV-cache init. Build + # the wheel via Mooncake's own scripts/build_wheel.sh (auditwheel bundles + # every .so into a self-contained package) and pip install it so the + # `mooncake` package lands in site-packages and is importable everywhere. + if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then + MOONCAKE_SRC_DIR="${MOONCAKE_SRC_DIR:-/opt/mooncake-src}" + MOONCAKE_GIT_REF="${MOONCAKE_GIT_REF:-main}" + MOONCAKE_PYVER="$(python3 -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + pip install --quiet build auditwheel patchelf + rm -rf "$MOONCAKE_SRC_DIR" + git clone https://github.com/kvcache-ai/Mooncake.git "$MOONCAKE_SRC_DIR" + ( cd "$MOONCAKE_SRC_DIR" + git checkout "$MOONCAKE_GIT_REF" + bash dependencies.sh + mkdir -p build && cd build + cmake .. -DWITH_STORE=ON + make -j + make install + cd .. + # Produces a self-contained wheel under mooncake-wheel/dist/. + bash scripts/build_wheel.sh "$MOONCAKE_PYVER" + pip install --force-reinstall mooncake-wheel/dist/*.whl + + # CMake installs the Python package into the user site, which + # shadows the wheel installed by pip. Ensure the CLI wrapper can + # find the native master binary in the package it imports. + MOONCAKE_PACKAGE_DIR=$(python3 -c 'import pathlib, mooncake; print(pathlib.Path(mooncake.__file__).parent)') + install -m 0755 build/mooncake-store/src/mooncake_master \ + "$MOONCAKE_PACKAGE_DIR/mooncake_master" ) + python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null + test -x "$(python3 -c 'import pathlib, mooncake; print(pathlib.Path(mooncake.__file__).parent / "mooncake_master")')" + fi + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + echo "Mooncake master PID: $MOONCAKE_MASTER_PID" + sleep 10 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup. Log follows:" >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 || true + exit 1 + fi + + PREFIX_CACHE_ARGS=(--enable-prefix-caching) + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + ;; + *) + echo "Error: unsupported KV_OFFLOAD_BACKEND '$OFFLOAD_MODE' (expected: lmcache, mooncake)" >&2 + exit 1 + ;; +esac + +EP_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + +# Parallelism layout: +# DP_ATTENTION=false -> pure TP: attention TP-sharded across all $TP GPUs in a +# single engine (low TPOT, capacity-capped at high CONC). +# DP_ATTENTION=true -> DEP: per-DP-rank attention (TP1 x DP=$TP) with experts +# EP-sharded across the DP ranks (requires EP_SIZE>1). Grows KV capacity + +# decode width with concurrency. Mirrors dsv4_fp4_b300_vllm.sh. +PARALLEL_ARGS=(--tensor-parallel-size="$TP") +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size=1 --data-parallel-size="$TP") + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + # vllm-router expands the one HTTP backend into one logical worker per DP + # rank and sends X-data-parallel-rank per request; consistent_hash pins each + # conversation to a rank so its prefix cache stays warm. aiperf's stable + # X-Correlation-ID is aliased to the router's X-Session-ID for that affinity. + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + pip install --quiet "vllm-router==0.1.14" +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 +export VLLM_SPEC_CONFIG="${VLLM_SPEC_CONFIG:-{\"model\": \"lightseekorg/kimi-k2.6-eagle3.1-mla\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3}}" + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + "${PARALLEL_ARGS[@]}" + "${EP_ARGS[@]}" + --gpu-memory-utilization 0.90 + --block-size=1 + --trust-remote-code + --max-num-seqs "$CONC" + --mm-encoder-tp-mode data + --speculative-config "$VLLM_SPEC_CONFIG" + "${PREFIX_CACHE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# In DEP mode, front the DP engine with vllm-router on $PORT so the agentic +# client (which targets $PORT) load-balances across DP ranks with prefix +# affinity. Pure-TP serves the client directly on $PORT. +if [ "$USE_VLLM_ROUTER" = "true" ]; then + ROUTER_LOG="$RESULT_DIR/router.log" + echo "Starting vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy consistent_hash \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +# ---- Run benchmark ---------------------------------------------------------- +build_replay_cmd "$RESULT_DIR" + +run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d829459643..05b5cd24e4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -548,6 +548,27 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] } +kimik2.7-fp4-mi355x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:v0.24.0 + model: amd/Kimi-K2.7-Code-MXFP4 + model-prefix: kimik2.7 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + # TP8, 3600s. Best-config sweep on the agentic trace, one KV tier per regime: + # GPU-only (none) is only viable at low conc [4,8] (peaks ~26.7 tok/s/GPU at + # c8, then collapses at the KV-capacity wall); LMCache DRAM offload carries the + # high-conc side and holds throughput where none thrashes, so it is swept at + # c16 (the crossover where it overtakes none). Mooncake and c32/48 dropped: + # they added no throughput headroom over this best-config envelope. + - dram-utilization: 0.80 + search-space: + - { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [16], spec-decoding: mtp } + kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539 model: amd/Kimi-K2.5-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 005967fc36..f5251f9713 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4966,3 +4966,11 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - kimik2.7-fp4-mi355x-vllm-agentic-mtp + description: + - "Add Kimi-K2.7-Code-MXFP4 agentic-coding MTP config on MI355X via vLLM v0.24.0, using EAGLE3 speculative decoding with lightseekorg/kimi-k2.6-eagle3.1-mla" + - "Mirror the best-config agentic sweep: GPU-only none at conc [4,8] and dram/LMCache at conc [16], duration 3600s, dram-utilization 0.80" + - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh reuses the Kimi-K2.7 agentic recipe and adds --speculative-config for EAGLE3" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER From 4f3a2dbf309fa374ab7c420e14227e49ba161131 Mon Sep 17 00:00:00 2001 From: haic0 <149741444+haic0@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:27:11 +0800 Subject: [PATCH 2/9] Update kimik2.7_fp4_mi355x_mtp.sh --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index b39e89a333..214da3f479 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -345,7 +345,6 @@ VLLM_CMD=( "${PARALLEL_ARGS[@]}" "${EP_ARGS[@]}" --gpu-memory-utilization 0.90 - --block-size=1 --trust-remote-code --max-num-seqs "$CONC" --mm-encoder-tp-mode data From f75d0251afb3bd87b8ed4e62dead3bce122384ad Mon Sep 17 00:00:00 2001 From: haic0 <149741444+haic0@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:27:46 +0800 Subject: [PATCH 3/9] Update kimik2.7_fp4_mi355x_mtp.sh --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index 214da3f479..bae509d8e3 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -346,7 +346,6 @@ VLLM_CMD=( "${EP_ARGS[@]}" --gpu-memory-utilization 0.90 --trust-remote-code - --max-num-seqs "$CONC" --mm-encoder-tp-mode data --speculative-config "$VLLM_SPEC_CONFIG" "${PREFIX_CACHE_ARGS[@]}" From 5ee1f3c9eaeeb7353f69264b6e375566a154197b Mon Sep 17 00:00:00 2001 From: haic0 Date: Mon, 20 Jul 2026 22:44:11 -0400 Subject: [PATCH 4/9] Fix Kimi MTP changelog base Restore upstream perf-changelog entries so the Kimi MTP branch remains append-only against current main. Signed-off-by: haic0 Co-authored-by: Cursor --- perf-changelog.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f5251f9713..6eeff6eba7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4887,6 +4887,14 @@ - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Add DeepSeek-V4-Pro FP4 MI355X SGLang-disagg agentic-coding benchmark with HiCache DRAM KV offloading." + - "Add the multi-node agentic runtime plumbing: node-0 sibling benchmark-client container, per-worker metrics scrape, per-concurrency cache flushing, trace replay client, and one allocation per concurrency." + - "Image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2170 + - config-keys: - dsv4-fp4-b200-vllm-agentic description: @@ -4967,6 +4975,14 @@ - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Convert the high-throughput arm to DEP: add ep: 8 to the TP8/DP8 attention-DP arm so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8) instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1). Attention stays data-parallel behind sglang-router consistent hashing; the recipe already passes --ep-size EP_SIZE unconditionally, so only the master-config search space changes" + - "DEP arm runs conc-list [48] only (was [48, 64, 96, 128, 192, 256, 512]): conc 48 is the frontier peak (163 tok/s/gpu out, ttft 2.8s); conc >=64 overflows the HBM+host radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is strictly dominated" + - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281 + - config-keys: - kimik2.7-fp4-mi355x-vllm-agentic-mtp description: From 8f29cf2566317ddff7846416d17bae82b4f7c493 Mon Sep 17 00:00:00 2001 From: haic0 Date: Mon, 20 Jul 2026 23:15:54 -0400 Subject: [PATCH 5/9] Update Kimi MTP synthetic spec config Use the synthetic EAGLE3 acceptance settings required for the Kimi-K2.7 MI355X agentic MTP sweep. Signed-off-by: haic0 Co-authored-by: Cursor --- .../single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index bae509d8e3..cf0f95d537 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -335,7 +335,17 @@ fi echo "Starting vllm server..." export PYTHONNOUSERSITE=1 -export VLLM_SPEC_CONFIG="${VLLM_SPEC_CONFIG:-{\"model\": \"lightseekorg/kimi-k2.6-eagle3.1-mla\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3}}" +DEFAULT_VLLM_SPEC_CONFIG=$(cat <<'EOF' +{ + "method": "eagle3", + "model": "lightseekorg/kimi-k2.6-eagle3.1-mla", + "num_speculative_tokens": 4, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_length": 3.24 +} +EOF +) +export VLLM_SPEC_CONFIG="${VLLM_SPEC_CONFIG:-$DEFAULT_VLLM_SPEC_CONFIG}" { set +x; } 2>/dev/null VLLM_CMD=( From bc153604b8075897c9081c5157d969be6fc9cb6f Mon Sep 17 00:00:00 2001 From: haic0 Date: Tue, 21 Jul 2026 06:28:55 -0400 Subject: [PATCH 6/9] Update Kimi MTP vLLM smoke settings Use the nightly ROCm vLLM image and tuned EAGLE3 synthetic settings that allow the LMCache smoke path to complete. Signed-off-by: haic0 Co-authored-by: Cursor --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 7 +++++-- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index cf0f95d537..97a582f0e0 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -339,9 +339,9 @@ DEFAULT_VLLM_SPEC_CONFIG=$(cat <<'EOF' { "method": "eagle3", "model": "lightseekorg/kimi-k2.6-eagle3.1-mla", - "num_speculative_tokens": 4, + "num_speculative_tokens": 3, "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 3.24 + "synthetic_acceptance_length": 2.88 } EOF ) @@ -357,6 +357,9 @@ VLLM_CMD=( --gpu-memory-utilization 0.90 --trust-remote-code --mm-encoder-tp-mode data + --block-size=1 + --max-num-seqs "$CONC" + --max-model-len auto --speculative-config "$VLLM_SPEC_CONFIG" "${PREFIX_CACHE_ARGS[@]}" "${OFFLOAD_ARGS[@]}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e42e8fd7d9..66b40e0dcd 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -549,7 +549,7 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] } kimik2.7-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:v0.24.0 + image: vllm/vllm-openai-rocm:nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1 model: amd/Kimi-K2.7-Code-MXFP4 model-prefix: kimik2.7 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eeff6eba7..871e0db9b5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4986,7 +4986,7 @@ - config-keys: - kimik2.7-fp4-mi355x-vllm-agentic-mtp description: - - "Add Kimi-K2.7-Code-MXFP4 agentic-coding MTP config on MI355X via vLLM v0.24.0, using EAGLE3 speculative decoding with lightseekorg/kimi-k2.6-eagle3.1-mla" + - "Add Kimi-K2.7-Code-MXFP4 agentic-coding MTP config on MI355X via vLLM nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1, using EAGLE3 speculative decoding with lightseekorg/kimi-k2.6-eagle3.1-mla" - "Mirror the best-config agentic sweep: GPU-only none at conc [4,8] and dram/LMCache at conc [16], duration 3600s, dram-utilization 0.80" - - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh reuses the Kimi-K2.7 agentic recipe and adds --speculative-config for EAGLE3" + - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh reuses the Kimi-K2.7 agentic recipe and adds EAGLE3 synthetic acceptance (3 speculative tokens, AL 2.88), block-size=1, max-model-len=auto, and max-num-seqs=CONC" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER From c122e50e8d28915e3f90892bd0edf8650d0c0c6d Mon Sep 17 00:00:00 2001 From: haic0 Date: Wed, 22 Jul 2026 06:44:12 -0400 Subject: [PATCH 7/9] Enable Kimi MTP agentic eval-only path Route eval-only runs through run_eval so run-sweep agentic eval jobs produce SWE-bench artifacts. Signed-off-by: haic0 Co-authored-by: Cursor --- .../single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index 97a582f0e0..d70f16c9c9 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -393,7 +393,10 @@ if [ "$USE_VLLM_ROUTER" = "true" ]; then wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" fi -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" +# ---- Run benchmark / eval --------------------------------------------------- +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi From 3d11eef36c6c778f3dc411e3872846b60518e22a Mon Sep 17 00:00:00 2001 From: haic0 Date: Wed, 22 Jul 2026 22:35:15 -0400 Subject: [PATCH 8/9] Enable Kimi MTP tool calling for evals Add Kimi tool-call and reasoning parsers with auto tool choice so agentic eval requests using tool_choice=auto are accepted by vLLM. Signed-off-by: haic0 Co-authored-by: Cursor --- benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh | 3 +++ perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index d70f16c9c9..2687b70eef 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -357,6 +357,9 @@ VLLM_CMD=( --gpu-memory-utilization 0.90 --trust-remote-code --mm-encoder-tp-mode data + --tool-call-parser kimi_k2 + --enable-auto-tool-choice + --reasoning-parser kimi_k2 --block-size=1 --max-num-seqs "$CONC" --max-model-len auto diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ddbae723fd..2e28357a9f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5055,5 +5055,5 @@ description: - "Add Kimi-K2.7-Code-MXFP4 agentic-coding MTP config on MI355X via vLLM nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1, using EAGLE3 speculative decoding with lightseekorg/kimi-k2.6-eagle3.1-mla" - "Mirror the best-config agentic sweep: GPU-only none at conc [4,8] and dram/LMCache at conc [16], duration 3600s, dram-utilization 0.80" - - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh reuses the Kimi-K2.7 agentic recipe and adds EAGLE3 synthetic acceptance (3 speculative tokens, AL 2.88), block-size=1, max-model-len=auto, and max-num-seqs=CONC" + - "Recipe benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh reuses the Kimi-K2.7 agentic recipe and adds EAGLE3 synthetic acceptance (3 speculative tokens, AL 2.88), Kimi tool-call/reasoning parsers with auto tool choice for agentic eval, block-size=1, max-model-len=auto, and max-num-seqs=CONC" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER From 4645afc0fbf8a65c7c83c981b7c08f4e775131be Mon Sep 17 00:00:00 2001 From: haic0 Date: Fri, 24 Jul 2026 05:55:08 -0400 Subject: [PATCH 9/9] Update Kimi MTP eval tool handling Signed-off-by: haic0 Co-authored-by: Cursor --- benchmarks/benchmark_lib.sh | 30 +- .../agentic/kimik2.7_fp4_mi355x_mtp.sh | 4 + configs/amd-master.yaml | 12 +- utils/evals/test_run_eval_dispatch.py | 8 +- utils/local_kimik27_agentic_mtp_eval_smoke.sh | 286 ++++++++++++++++++ 5 files changed, 323 insertions(+), 17 deletions(-) create mode 100755 utils/local_kimik27_agentic_mtp_eval_smoke.sh diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 9a01f63b4b..410a73ab50 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1192,6 +1192,15 @@ default_path, out_path = sys.argv[1], sys.argv[2] d = yaml.safe_load(open(default_path)) or {} d.setdefault("agent", {}) step_limit = int(os.environ.get("SWEBENCH_AGENT_STEP_LIMIT", "75")) +model_name = os.environ.get("MODEL_NAME") or os.environ.get("MODEL", "") +model_prefix = os.environ.get("MODEL_PREFIX", "") +kimi_tool_guidance = "" +if model_prefix.startswith("kimik") or "Kimi" in model_name: + kimi_tool_guidance = """ +- This model uses Kimi-native tool-call markers behind the OpenAI tool API. Do not answer in prose only. +- Every assistant turn must call the bash tool through the provided tool interface. Use exactly one bash tool call per turn. +- If you emit native Kimi markers, the tool id must be `functions.bash:` and the arguments must be valid JSON for the bash tool. Do not invent other tool names. +- After each observation, continue with another bash tool call until the patch is ready. A response without a tool call is a format error and scores ZERO.""" guidance = f""" @@ -1200,6 +1209,7 @@ guidance = f""" - The scoring harness re-runs tests in its own clean environment. If the package fails to BUILD or IMPORT after 2-3 attempts, do NOT keep fixing the environment -- apply your source-code fix and submit it. A local build is not required for your patch to score. - `git diff` alone is NOT a submission. Submitting requires the exact final command sequence described above. - When unsure how code behaves, write and RUN a short script instead of reasoning about it at length in prose. +{kimi_tool_guidance} """ it = d["agent"].get("instance_template", "") d["agent"]["instance_template"] = it.rstrip() + guidance + "\n" @@ -1218,16 +1228,22 @@ agent_cpu = os.environ.get("SWEBENCH_AGENT_SANDBOX_CPU", "") if agent_cpu: env["modal_sandbox_kwargs"] = {"cpu": float(agent_cpu)} d["environment"] = env -model_name = os.environ.get("MODEL_NAME") or os.environ.get("MODEL", "") +model_kwargs = { + "api_base": f"http://0.0.0.0:{os.environ['SWEBENCH_AGENT_PORT']}/v1", + "api_key": "dummy", + "custom_llm_provider": "openai", + "temperature": 0.0, +} +tool_choice = os.environ.get("SWEBENCH_AGENT_TOOL_CHOICE", "") +if tool_choice: + model_kwargs["tool_choice"] = tool_choice +parallel_tool_calls = os.environ.get("SWEBENCH_AGENT_PARALLEL_TOOL_CALLS", "") +if parallel_tool_calls: + model_kwargs["parallel_tool_calls"] = parallel_tool_calls.lower() not in {"0", "false", "no", "off"} d["model"] = { "model_name": f"openai/{model_name}", "cost_tracking": "ignore_errors", - "model_kwargs": { - "api_base": f"http://0.0.0.0:{os.environ['SWEBENCH_AGENT_PORT']}/v1", - "api_key": "dummy", - "custom_llm_provider": "openai", - "temperature": 0.0, - }, + "model_kwargs": model_kwargs, } yaml.safe_dump(d, open(out_path, "w"), default_flow_style=False, sort_keys=False) PYGEN diff --git a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh index 2687b70eef..f67afbb42e 100755 --- a/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh @@ -398,6 +398,10 @@ fi # ---- Run benchmark / eval --------------------------------------------------- if [ "${EVAL_ONLY}" = "true" ]; then + # Kimi's forced-tool path currently trips vLLM/xgrammar; keep auto tool + # choice and rely on Kimi-specific agent guidance to select the bash tool. + export SWEBENCH_AGENT_TOOL_CHOICE="${SWEBENCH_AGENT_TOOL_CHOICE:-auto}" + export SWEBENCH_AGENT_PARALLEL_TOOL_CALLS="${SWEBENCH_AGENT_PARALLEL_TOOL_CALLS:-false}" run_eval --port "$PORT" else build_replay_cmd "$RESULT_DIR" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6f9ce521f0..cad4a46b87 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -558,16 +558,12 @@ kimik2.7-fp4-mi355x-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - # TP8, 3600s. Best-config sweep on the agentic trace, one KV tier per regime: - # GPU-only (none) is only viable at low conc [4,8] (peaks ~26.7 tok/s/GPU at - # c8, then collapses at the KV-capacity wall); LMCache DRAM offload carries the - # high-conc side and holds throughput where none thrashes, so it is swept at - # c16 (the crossover where it overtakes none). Mooncake and c32/48 dropped: - # they added no throughput headroom over this best-config envelope. + # TP4, 3600s. Sweep LMCache DRAM across the agentic concurrency ladder and + # keep a single GPU-only MTP point for comparison. - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [4, 8], spec-decoding: mtp } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [16], spec-decoding: mtp } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "aaf7c0d3" }, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], spec-decoding: mtp } + - { tp: 4, kv-offloading: none, conc-list: [4], spec-decoding: mtp } kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.4_202607091539 diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index d0540b5cc6..7608d23585 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -417,11 +417,15 @@ def test_agentic_generation_invokes_mini_swe_agent(tmp_path): source "$BENCHMARK_LIB" 2>/dev/null _install_swebench_agent_deps() { :; } _ensure_modal_credentials() { :; } -export EVAL_LIMIT=10 MODEL_NAME=test-model SWEBENCH_SANDBOX_SWEEP=0 SWEBENCH_WATCHDOG_POLL=1 +export EVAL_LIMIT=10 MODEL_NAME=amd/Kimi-K2.7-Code-MXFP4 MODEL_PREFIX=kimik2.7 SWEBENCH_SANDBOX_SWEEP=0 SWEBENCH_WATCHDOG_POLL=1 +export SWEBENCH_AGENT_TOOL_CHOICE=auto SWEBENCH_AGENT_PARALLEL_TOOL_CALLS=false _run_swebench_agentic_generation "$GEN_DIR" --port 8899 || exit 1 [ -s "$GEN_DIR/agent_out/preds.json" ] || { echo NO_PREDS; exit 1; } grep -q 'api_base: http://0.0.0.0:8899/v1' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo BAD_PORT; exit 1; } -grep -q 'openai/test-model' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo BAD_MODEL; exit 1; } +grep -q 'openai/amd/Kimi-K2.7-Code-MXFP4' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo BAD_MODEL; exit 1; } +grep -q 'tool_choice: auto' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_TOOL_CHOICE; exit 1; } +grep -q 'parallel_tool_calls: false' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo BAD_PARALLEL_TOOLS; exit 1; } +grep -q 'Kimi-native tool-call markers' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_KIMI_GUIDANCE; exit 1; } grep -q 'additional_critical_guidance' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_GUIDANCE; exit 1; } grep -q 'BEFORE submitting you MUST run the test' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_VERIFY_RULE; exit 1; } grep -q 'runtime_timeout: 3600' "$GEN_DIR/mini_swebench_overrides.yaml" || { echo NO_RUNTIME_TIMEOUT; exit 1; } diff --git a/utils/local_kimik27_agentic_mtp_eval_smoke.sh b/utils/local_kimik27_agentic_mtp_eval_smoke.sh new file mode 100755 index 0000000000..1770044089 --- /dev/null +++ b/utils/local_kimik27_agentic_mtp_eval_smoke.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: utils/local_kimik27_agentic_mtp_eval_smoke.sh [options] + +Simulate the run-sweep.yml agentic eval-only path for the Kimi-K2.7 FP4 +MI355X vLLM MTP recipe. This sets the same benchmark-tmpl.yml env shape, +runs the Kimi benchmark with EVAL_ONLY=true, then performs the eval artifact +checks that the GitHub workflow performs. + +By default this is a dry run. Use --execute from an MI355X container/allocation +with vLLM, ROCm, HF credentials, Modal or local SWE-bench scoring access, and +the benchmark dependencies available. + +Options: + --row ROW none-c4 | none-c8 | lmcache-c16 | none-c1 | lmcache-c1 + (default: lmcache-c16; matches the failing eval-only row) + --eval-limit N SWE-bench slice size for smoke testing (default: 1) + --result-dir DIR Result directory (default: results/local_kimik27_agentic_mtp_eval_ROW) + --gen-mode MODE SWE-bench generation mode: agentic | single-shot (default: agentic) + --use-modal Score with Modal sandboxes, matching CI (default) + --no-modal Score locally with SWE-bench Docker instead of Modal + --skip-score Generate predictions only; skips results*.json and score validation + --validate-scores Run utils/evals/validate_scores.py after the script finishes + --execute Run the benchmark script. Without this, only print env. + -h, --help Show this help. + +Examples: + # Inspect the CI-like eval environment without running. + utils/local_kimik27_agentic_mtp_eval_smoke.sh + + # Run one Kimi+LMCache SWE-bench instance through the eval flow. + utils/local_kimik27_agentic_mtp_eval_smoke.sh --execute --eval-limit 1 + + # Run generation only to debug mini-swe-agent tool-call behavior quickly. + utils/local_kimik27_agentic_mtp_eval_smoke.sh --execute --skip-score --eval-limit 1 +USAGE +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +row="lmcache-c16" +eval_limit="1" +result_dir="" +gen_mode="agentic" +use_modal="true" +skip_score="false" +validate_scores="false" +execute="false" + +while [[ $# -gt 0 ]]; do + case "$1" in + --row) + row="${2:?--row requires a value}" + shift 2 + ;; + --eval-limit) + eval_limit="${2:?--eval-limit requires a value}" + shift 2 + ;; + --result-dir) + result_dir="${2:?--result-dir requires a value}" + shift 2 + ;; + --gen-mode) + gen_mode="${2:?--gen-mode requires a value}" + shift 2 + ;; + --use-modal) + use_modal="true" + shift + ;; + --no-modal) + use_modal="false" + shift + ;; + --skip-score) + skip_score="true" + shift + ;; + --validate-scores) + validate_scores="true" + shift + ;; + --execute) + execute="true" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +case "$row" in + none-c1) + conc=1 + kv_offloading="none" + kv_offload_backend="" + kv_offload_backend_metadata="" + total_cpu_dram_gb=0 + ;; + none-c4) + conc=4 + kv_offloading="none" + kv_offload_backend="" + kv_offload_backend_metadata="" + total_cpu_dram_gb=0 + ;; + none-c8) + conc=8 + kv_offloading="none" + kv_offload_backend="" + kv_offload_backend_metadata="" + total_cpu_dram_gb=0 + ;; + lmcache-c1) + conc=1 + kv_offloading="dram" + kv_offload_backend="lmcache" + kv_offload_backend_metadata='{"name":"lmcache","version":"aaf7c0d3"}' + total_cpu_dram_gb="${TOTAL_CPU_DRAM_GB:-2399}" + ;; + lmcache-c16) + conc=16 + kv_offloading="dram" + kv_offload_backend="lmcache" + kv_offload_backend_metadata='{"name":"lmcache","version":"aaf7c0d3"}' + total_cpu_dram_gb="${TOTAL_CPU_DRAM_GB:-2399}" + ;; + *) + echo "Unsupported row '$row'" >&2 + usage >&2 + exit 2 + ;; +esac + +if [[ "$eval_limit" != "full" && "$eval_limit" != "0" && ! "$eval_limit" =~ ^[1-9][0-9]*$ ]]; then + echo "--eval-limit must be a positive integer, 'full', or 0; got '$eval_limit'" >&2 + exit 2 +fi + +case "$gen_mode" in + agentic|single-shot) ;; + *) + echo "--gen-mode must be 'agentic' or 'single-shot', got '$gen_mode'" >&2 + exit 2 + ;; +esac + +tp="${TP:-8}" +ep_size="${EP_SIZE:-1}" +dp_attention="${DP_ATTENTION:-false}" + +export RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-0.8}" +export SPEC_DECODING="mtp" +export EXP_NAME="kimik2.7_tp${tp}_conc${conc}_kv${kv_offloading}${kv_offload_backend:+-${kv_offload_backend}}_spec-${SPEC_DECODING}" +export MODEL="amd/Kimi-K2.7-Code-MXFP4" +export MODEL_PREFIX="kimik2.7" +export IMAGE="${IMAGE:-vllm/vllm-openai-rocm:nightly-f25953cc59f9b4ba9b04b16228d2b86dcfbcbdb1}" +export FRAMEWORK="vllm" +export PRECISION="fp4" +export TP="$tp" +export PP_SIZE="${PP_SIZE:-1}" +export DCP_SIZE="${DCP_SIZE:-1}" +export PCP_SIZE="${PCP_SIZE:-1}" +export EP_SIZE="$ep_size" +export DP_ATTENTION="$dp_attention" +export CONC="$conc" +export DISAGG="false" +export RUN_EVAL="true" +export EVAL_ONLY="true" +export RUNNER_TYPE="${RUNNER_TYPE:-cluster:mi355x-amds}" +export RUNNER_NAME="${RUNNER_NAME:-local_eval}" +export IS_MULTINODE="${IS_MULTINODE:-false}" +export SCENARIO_TYPE="agentic-coding" +export SCENARIO_SUBDIR="agentic/" +export IS_AGENTIC="1" +export KV_OFFLOADING="$kv_offloading" +export KV_OFFLOAD_BACKEND="$kv_offload_backend" +export KV_OFFLOAD_BACKEND_METADATA="$kv_offload_backend_metadata" +export ROUTER_METADATA="${ROUTER_METADATA:-}" +export KV_P2P_TRANSFER="${KV_P2P_TRANSFER:-}" +export TOTAL_CPU_DRAM_GB="$total_cpu_dram_gb" +export DURATION="${DURATION:-3600}" +export EVAL_LIMIT="$eval_limit" +export SWEBENCH_GEN_MODE="$gen_mode" +export SWEBENCH_USE_MODAL="$use_modal" +export SWEBENCH_SKIP_SCORE="$skip_score" +export SWEBENCH_AGENT_TOOL_CHOICE="${SWEBENCH_AGENT_TOOL_CHOICE:-auto}" +export SWEBENCH_AGENT_PARALLEL_TOOL_CALLS="${SWEBENCH_AGENT_PARALLEL_TOOL_CALLS:-false}" +export RESULT_DIR="${result_dir:-$repo_root/results/local_kimik27_agentic_mtp_eval_${row}}" +export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-$repo_root}" +export AIPERF_FAILED_REQUEST_THRESHOLD="${AIPERF_FAILED_REQUEST_THRESHOLD:-0.10}" +export HF_HUB_CACHE="${HF_HUB_CACHE:-$repo_root/.cache/hf_hub}" +export AIPERF_DATASET_MMAP_CACHE_DIR="${AIPERF_DATASET_MMAP_CACHE_DIR:-$repo_root/.cache/aiperf_mmap}" +export PYTHONDONTWRITEBYTECODE=1 +export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-/tmp/inferencex-pycache}" +export PORT="${PORT:-8888}" +export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE)) + +script="benchmarks/single_node/agentic/kimik2.7_fp4_mi355x_mtp.sh" +script_path="$repo_root/$script" +result_filename="${EXP_NAME}_${PRECISION}_${FRAMEWORK}_tp${TP}-pp${PP_SIZE}-dcp${DCP_SIZE}-pcp${PCP_SIZE}-ep${EP_SIZE}-dpa${DP_ATTENTION}_disagg-${DISAGG}_spec-${SPEC_DECODING}_conc${CONC}_${RUNNER_NAME}" +export RESULT_FILENAME="${RESULT_FILENAME:-$result_filename}" + +cat <} + KV_OFFLOAD_BACKEND_METADATA=${KV_OFFLOAD_BACKEND_METADATA:-} + EVAL_ONLY=$EVAL_ONLY RUN_EVAL=$RUN_EVAL EVAL_LIMIT=$EVAL_LIMIT SWEBENCH_GEN_MODE=$SWEBENCH_GEN_MODE + SWEBENCH_USE_MODAL=$SWEBENCH_USE_MODAL SWEBENCH_SKIP_SCORE=$SWEBENCH_SKIP_SCORE + SWEBENCH_AGENT_TOOL_CHOICE=$SWEBENCH_AGENT_TOOL_CHOICE + SWEBENCH_AGENT_PARALLEL_TOOL_CALLS=$SWEBENCH_AGENT_PARALLEL_TOOL_CALLS + RESULT_DIR=$RESULT_DIR +EOF + +if [[ ! -f "$script_path" ]]; then + cat >&2 <&2 + exit 1 + fi + echo "SWEBENCH_SKIP_SCORE=true: generation artifacts were produced; skipping results*.json and score validation." +else + if ! ls results*.json >/dev/null 2>&1; then + echo "Eval smoke failed: no results*.json files found." >&2 + exit 1 + fi + if [[ "$validate_scores" == "true" ]]; then + python3 utils/evals/validate_scores.py + else + echo "Skipping score threshold validation; pass --validate-scores to mirror the final CI gate." + fi +fi + +cat </dev/null || true) +EOF