@@ -3,8 +3,6 @@ set -euo pipefail
33set -x
44
55# Agentic trace replay benchmark for Kimi-K2.7 FP4 on MI355X using vLLM.
6- #
7- # Variant of kimik2.5_fp4_mi355x.sh that supports TWO KV configs:
86# KV_OFFLOADING=none -> GPU KV only
97# KV_OFFLOADING=dram KV_OFFLOAD_BACKEND=lmcache -> LMCache MP server + connector
108#
@@ -25,9 +23,6 @@ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then
2523 export HIP_VISIBLE_DEVICES=" $ROCR_VISIBLE_DEVICES "
2624fi
2725
28- # `hf download` creates the target dir if missing and is itself idempotent.
29- # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
30- # Either way, MODEL_PATH is what the server is launched with.
3126if [[ -n " ${MODEL_PATH:- } " ]]; then
3227 if [[ ! -d " $MODEL_PATH " || -z " $( ls -A " $MODEL_PATH " 2> /dev/null) " ]]; then
3328 hf download " $MODEL " --local-dir " $MODEL_PATH "
@@ -120,8 +115,6 @@ wait_for_lmcache_ready() {
120115 exit 1
121116}
122117
123- # Resolve the effective offload backend. When KV_OFFLOADING=none there is no
124- # backend; when dram, KV_OFFLOAD_BACKEND selects native vs lmcache.
125118if [[ " $KV_OFFLOADING " == " none" ]]; then
126119 OFFLOAD_MODE=" none"
127120else
130123
131124case " $OFFLOAD_MODE " in
132125 none)
133- # GPU-only KV, no DRAM offload. Leave prefix caching at vLLM's
134- # default (ON) so this is an honest no-offload baseline that still
135- # reuses shared prefixes on-GPU — apples-to-apples vs the lmcache
136- # cell, which extends that same reuse into DRAM. (Matches the
137- # kimik2.5 / dsv4 agentic recipes.)
126+ # GPU-only KV baseline: keep on-GPU prefix caching ON (no DRAM offload)
127+ # so it's apples-to-apples vs the lmcache cell.
138128 PREFIX_CACHE_ARGS=(--enable-prefix-caching)
139129 ;;
140130 lmcache)
141131 unset VLLM_USE_SIMPLE_KV_OFFLOAD
142132
143- # Build LMCache against ROCm if the connector isn't already importable
144- # (prebuilt kimi-lmcache images already ship it). Clone to a
145- # container-local dir (NOT the bind-mounted /workspace) so the CI
146- # checkout's `clean: true` never trips over root-owned build artifacts
147- # on the next job. Pin a ref for reproducibility.
133+ # Build LMCache against ROCm if the connector isn't importable. Clone to
134+ # a container-local dir (NOT bind-mounted /workspace) so the next job's
135+ # checkout `clean: true` won't trip over root-owned build artifacts.
148136 if ! python3 -c " import lmcache.integration.vllm.lmcache_mp_connector" > /dev/null 2>&1 ; then
149137 LMCACHE_SRC_DIR=" ${LMCACHE_SRC_DIR:-/ opt/ lmcache-src} "
150138 LMCACHE_GIT_REF=" ${LMCACHE_GIT_REF:- aaf7c0d3} "
@@ -161,11 +149,9 @@ case "$OFFLOAD_MODE" in
161149 LMCACHE_PORT=" ${LMCACHE_PORT:- 5555} "
162150 LMCACHE_HTTP_PORT=" ${LMCACHE_HTTP_PORT:- 8080} "
163151 LMCACHE_CONNECT_HOST=" ${LMCACHE_CONNECT_HOST:- tcp:// $LMCACHE_HOST } "
164- # Let the external MP server own the whole CPU KV pool. The requested
165- # budget is TOTAL_CPU_DRAM_GB, but LMCache's L1 is SHM-backed: if L1 >
166- # /dev/shm free it silently disables SHM and falls back to the slow
167- # pickle path (crashes at load — see kimik27 CI shm-overflow note).
168- # Cap L1 to 90% of current /dev/shm free space so SHM stays enabled.
152+ # LMCache L1 is SHM-backed: if L1 > /dev/shm free it silently disables
153+ # SHM and falls back to the pickle path (crashes at load). Cap L1 to 90%
154+ # of /dev/shm free so SHM stays enabled.
169155 SHM_FREE_GB=$( df -BG --output=avail /dev/shm 2> /dev/null | tail -1 | tr -dc ' 0-9' )
170156 SHM_CAP_GB=$(( SHM_FREE_GB * 90 / 100 ))
171157 LMCACHE_L1_SIZE_GB=" ${LMCACHE_L1_SIZE_GB:- $TOTAL_CPU_DRAM_GB } "
0 commit comments