fix: clarify cl vs cache_length, add multi-run warning

huangzhenhua111 · huangzhenhua111 · commit 31fc935001b5 · 2026-02-25T21:13:48.000+08:00
- rename cl->ctx_len in scripts (was confusing with -cl flag) - print cache_length in output for clarity - force runs=1 for LLaMA only (clear() is no-op, Qwen3 unaffected) - update README with CTX_LENS and RUNS limitation note per @jialilve feedback
diff --git a/tools/mllm-llm-benchmark/README.md b/tools/mllm-llm-benchmark/README.md
@@ -53,16 +53,16 @@ Output goes to `bench_context/context_sweep_v2.csv`.
 - `MODEL`: Path to model file (required)
 - `CFG`: Path to config json (default: `./examples/llama/config_tiny_llama.json`)
 - `THREADS`: Number of threads (default: 8)
-- `RUNS`: How many runs to average (default: 1)
+- `RUNS`: How many runs to average (default: 1, LLaMA models auto-limited to 1)
 - `COOLDOWN`: Seconds to wait between runs (default: 0)
-- `CLS`: Context lengths to test (default: "256 512 1024 2048 4096")
+- `CTX_LENS`: Context lengths to test (default: "256 512 1024 2048 4096")
 - `TG_DH`: Generate length for decode_heavy mode (default: 256)
 - `TG_TTFT`: Generate length for prefill_ttft mode (default: 2)
 - `OUTDIR`: Output directory (default: bench_context)
 
 **Test modes:**
-- `prefill_ttft`: Measures time to first token (prompt length = CL-2, generates 2 tokens)
-- `decode_heavy`: Measures decode throughput (prompt length = CL-256, generates 256 tokens)
+- `prefill_ttft`: Measures time to first token (prompt length = CTX_LEN-2, generates 2 tokens)
+- `decode_heavy`: Measures decode throughput (prompt length = CTX_LEN-256, generates 256 tokens)
 
 ### Plot Results
 
diff --git a/tools/mllm-llm-benchmark/main.cpp b/tools/mllm-llm-benchmark/main.cpp
@@ -78,20 +78,32 @@ MLLM_MAIN({
     return 1;
   }
 
+  std::string m_name = model_name.get();
+  std::transform(m_name.begin(), m_name.end(), m_name.begin(),
+                 [](unsigned char c) { return std::tolower(c); });
+  if (R > 1 && (m_name.find("llama") != std::string::npos ||
+                m_name.find("tiny_llama") != std::string::npos)) {
+    mllm::print("[WARN] KV-cache reset not available for LLaMA yet");
+    mllm::print("[WARN] Forcing runs=1 to prevent cache corruption");
+    R = 1;
+
+  }
+
   std::ofstream csv_file;
   if (!output_csv.get().empty()) {
     csv_file.open(output_csv.get());
     if (!csv_file.is_open()) {
       mllm::print("[ERROR] Failed to open --output_csv:", output_csv.get());
       return 1;
     }
-    csv_file << "schema_version,git_commit,arch,model_name,pp,tg,ttft_ms,prefill_speed,decode_speed,prefill_ms,decode_ms_per_"
+    csv_file << "schema_version,git_commit,arch,model_name,cache_length,pp,tg,ttft_ms,prefill_speed,decode_speed,prefill_ms,decode_ms_per_"
                 "tok,kv_est_bytes_pp,kv_est_bytes_final\n";
   }
 
   mllm::print("Model Info");
   benchmark->init(config_path.get(), model_path.get(), cache_length.get());
   benchmark->printModelInfo();
+  mllm::print("Cache Length       :", cache_length.get());
 
   // Warmup run
   mllm::print("Warmup Run");
@@ -202,7 +214,7 @@ MLLM_MAIN({
 
     std::stringstream ss;
     ss << schema_version.get() << "," << STRINGIFY(MLLM_GIT_COMMIT_HASH) << "," << mllm::cpu::CURRENT_ARCH_STRING << ","
-       << model_name.get() << "," << pp << "," << tg << "," << avg_ttft << "," << avg_prefill_speed << "," << avg_decode_speed
+       << model_name.get() << "," << cache_length.get() << "," << pp << "," << tg << "," << avg_ttft << "," << avg_prefill_speed << "," << avg_decode_speed
        << "," << avg_prefill_ms << "," << avg_decode_ms_per_tok << "," << kv_est_bytes_pp << "," << kv_est_bytes_final;
 
     if (csv_file.is_open()) { csv_file << ss.str() << std::endl; }
diff --git a/tools/mllm-llm-benchmark/scripts/plot_sweep.py b/tools/mllm-llm-benchmark/scripts/plot_sweep.py
@@ -20,7 +20,7 @@ def to_int(x):
         rows.append(row)
 
 # normalize numeric fields
-num_fields = ["cl","pp","tg","threads","ttft_ms","prefill_ms","decode_ms","decode_ms_per_tok","peak_rss_kb","kv_est_kb"]
+num_fields = ["ctx_len","pp","tg","threads","ttft_ms","prefill_ms","decode_ms","decode_ms_per_tok","peak_rss_kb","kv_est_kb"]
 for row in rows:
     for k in num_fields:
         if k in row:
@@ -29,13 +29,13 @@ def to_int(x):
 # write summary
 stamp = os.path.splitext(os.path.basename(csv_path))[0]
 summary_path = os.path.join(out_dir, f"{stamp}.summary.csv")
-fieldnames = ["ts","git","arch","model","mode","cl","pp","tg","threads",
+fieldnames = ["ts","git","arch","model","mode","ctx_len","pp","tg","threads",
               "ttft_ms","prefill_ms","decode_ms","decode_ms_per_tok",
               "peak_rss_kb","kv_est_kb","peak_rss_gb","kv_est_mb"]
 with open(summary_path, "w", newline="") as f:
     w = csv.DictWriter(f, fieldnames=fieldnames)
     w.writeheader()
-    for row in sorted(rows, key=lambda x: (x.get("mode",""), x.get("cl",0))):
+    for row in sorted(rows, key=lambda x: (x.get("mode",""), x.get("ctx_len",0))):
         peak_rss_kb = row.get("peak_rss_kb", float("nan"))
         kv_est_kb   = row.get("kv_est_kb", float("nan"))
         out = {k: row.get(k, "") for k in fieldnames}
@@ -67,32 +67,32 @@ def plot_mode(mode, xkey, ykey, ylabel, fname):
     plt.savefig(os.path.join(out_dir, fname), dpi=180)
     plt.close()
 
-plot_mode("prefill_ttft", "cl", "ttft_ms", "TTFT (ms)", f"{stamp}.prefill_ttft.ttft_ms.png")
-plot_mode("prefill_ttft", "cl", "prefill_ms", "Prefill latency (ms)", f"{stamp}.prefill_ttft.prefill_ms.png")
-plot_mode("decode_heavy", "cl", "decode_ms_per_tok", "Decode latency per token (ms)", f"{stamp}.decode_heavy.decode_ms_per_tok.png")
-plot_mode("decode_heavy", "cl", "decode_ms", "Decode latency total (ms)", f"{stamp}.decode_heavy.decode_ms.png")
+plot_mode("prefill_ttft", "ctx_len", "ttft_ms", "TTFT (ms)", f"{stamp}.prefill_ttft.ttft_ms.png")
+plot_mode("prefill_ttft", "ctx_len", "prefill_ms", "Prefill latency (ms)", f"{stamp}.prefill_ttft.prefill_ms.png")
+plot_mode("decode_heavy", "ctx_len", "decode_ms_per_tok", "Decode latency per token (ms)", f"{stamp}.decode_heavy.decode_ms_per_tok.png")
+plot_mode("decode_heavy", "ctx_len", "decode_ms", "Decode latency total (ms)", f"{stamp}.decode_heavy.decode_ms.png")
 
 # memory plots
 mem = {}
 for row in rows:
-    cl = row.get("cl", float("nan"))
-    if cl != cl: 
+    ctx_len = row.get("ctx_len", float("nan"))
+    if ctx_len != ctx_len: 
         continue
-    cl = int(cl)
+    ctx_len = int(ctx_len)
     peak = row.get("peak_rss_kb", float("nan"))
     kv   = row.get("kv_est_kb", float("nan"))
-    cur = mem.get(cl, {"peak": float("nan"), "kv": float("nan")})
+    cur = mem.get(ctx_len, {"peak": float("nan"), "kv": float("nan")})
     if peak==peak and (cur["peak"]!=cur["peak"] or peak>cur["peak"]): cur["peak"]=peak
     if kv==kv and (cur["kv"]!=cur["kv"] or kv>cur["kv"]): cur["kv"]=kv
-    mem[cl]=cur
+    mem[ctx_len]=cur
 
-cls = sorted(mem.keys())
-peak_gb = [(mem[c]["peak"]/(1024*1024)) if mem[c]["peak"]==mem[c]["peak"] else float("nan") for c in cls]
-kv_mb   = [(mem[c]["kv"]/1024) if mem[c]["kv"]==mem[c]["kv"] else float("nan") for c in cls]
+ctx_lens = sorted(mem.keys())
+peak_gb = [(mem[c]["peak"]/(1024*1024)) if mem[c]["peak"]==mem[c]["peak"] else float("nan") for c in ctx_lens]
+kv_mb   = [(mem[c]["kv"]/1024) if mem[c]["kv"]==mem[c]["kv"] else float("nan") for c in ctx_lens]
 
 plt.figure()
-plt.plot(cls, peak_gb, marker="o")
-plt.xlabel("cl")
+plt.plot(ctx_lens, peak_gb, marker="o")
+plt.xlabel("ctx_len")
 plt.ylabel("Peak RSS (GB)")
 plt.xscale("log", base=2)
 plt.grid(True, which="both", linestyle="--", linewidth=0.5)
@@ -101,8 +101,8 @@ def plot_mode(mode, xkey, ykey, ylabel, fname):
 plt.close()
 
 plt.figure()
-plt.plot(cls, kv_mb, marker="o")
-plt.xlabel("cl")
+plt.plot(ctx_lens, kv_mb, marker="o")
+plt.xlabel("ctx_len")
 plt.ylabel("KV estimate (MB)")
 plt.xscale("log", base=2)
 plt.grid(True, which="both", linestyle="--", linewidth=0.5)
diff --git a/tools/mllm-llm-benchmark/scripts/sweep_context_v2.sh b/tools/mllm-llm-benchmark/scripts/sweep_context_v2.sh
@@ -9,13 +9,13 @@ RUNS="${RUNS:-1}"
 COOLDOWN="${COOLDOWN:-0}"
 TG_DH="${TG_DH:-256}"
 TG_TTFT="${TG_TTFT:-2}"
-CLS="${CLS:-256 512 1024 2048 4096}"
+CTX_LENS="${CTX_LENS:-256 512 1024 2048 4096}"
 OUTDIR="${OUTDIR:-bench_context}"
 
 mkdir -p "$OUTDIR"
 OUTCSV="$OUTDIR/context_sweep_v2.csv"
 
-echo "ts,git,arch,model,mode,cl,pp,tg,threads,ttft_ms,prefill_ms,decode_ms,decode_ms_per_tok,peak_rss_kb,kv_est_kb" > "$OUTCSV"
+echo "ts,git,arch,model,mode,ctx_len,pp,tg,threads,ttft_ms,prefill_ms,decode_ms,decode_ms_per_tok,peak_rss_kb,kv_est_kb" > "$OUTCSV"
 
 GIT="$(git rev-parse --short=12 HEAD 2>/dev/null || echo NA)"
 ARCH="$(uname -m)"
@@ -25,47 +25,48 @@ kv_est_kb() {
   python3 - <<'PY'
 import json, os, math
 cfg = os.environ["CFG"]
-cl = int(os.environ["CL"])
+ctx_len = int(os.environ["CTX_LEN"])
 bpe = int(os.environ.get("KV_BYTES_PER_ELEM","2"))
 j = json.load(open(cfg,"r"))
 L = int(j["num_hidden_layers"])
 H = int(j["num_attention_heads"])
 KVH = int(j.get("num_key_value_heads", H))
 hidden = int(j["hidden_size"])
 head_dim = hidden // H
-kv_bytes = 2 * L * KVH * head_dim * cl * bpe
+kv_bytes = 2 * L * KVH * head_dim * ctx_len * bpe
 print(int(math.ceil(kv_bytes / 1024)))
 PY
 }
 
 run_one () {
   local mode="$1"
-  local cl="$2"
+  local ctx_len="$2"
   local tg="$3"
 
-  if (( cl <= tg )); then
-    echo "skip: cl=$cl <= tg=$tg (mode=$mode)"
+  if (( ctx_len <= tg )); then
+    echo "skip: ctx_len=$ctx_len <= tg=$tg (mode=$mode)"
     return 0
   fi
 
-  local pp=$((cl - tg))
+  local pp=$((ctx_len - tg))
   if (( pp < 1 )); then pp=1; fi
 
-  echo "==== mode=$mode cl=$cl pp=$pp tg=$tg ===="
+  echo "==== mode=$mode ctx_len=$ctx_len pp=$pp tg=$tg ===="
 
-  local ALLLOG="$OUTDIR/run_${mode}_cl${cl}.all"
-  local TIMELOG="$OUTDIR/run_${mode}_cl${cl}.time"
+  local ALLLOG="$OUTDIR/run_${mode}_ctx${ctx_len}.all"
+  local TIMELOG="$OUTDIR/run_${mode}_ctx${ctx_len}.time"
 
   set +e
+  # pass ctx as cache limit
   /usr/bin/time -v \
     "$BIN" -n tiny_llama -m "$MODEL" -c "$CFG" \
-      -pp "$pp" -tg "$tg" -t "$THREADS" -cl "$cl" -r "$RUNS" -cs "$COOLDOWN" \
+      -pp "$pp" -tg "$tg" -t "$THREADS" -cl "$ctx_len" -r "$RUNS" -cs "$COOLDOWN" \
     >"$ALLLOG" 2>"$TIMELOG"
   local EXIT_CODE=$?
   set -e
 
   if [ $EXIT_CODE -ne 0 ]; then
-    echo "run failed with exit code $EXIT_CODE: mode=$mode cl=$cl"
+    echo "run failed with exit code $EXIT_CODE: mode=$mode ctx_len=$ctx_len"
     return 1
   fi
 
@@ -78,16 +79,16 @@ run_one () {
   DECODE_PER_TOK="$(python3 -c "tg=float('$tg'); d=float('$DECODE_MS'); print(d/tg if tg>0 else 0.0)")"
 
   PEAK_RSS_KB="$(grep -oP 'Maximum resident set size \(kbytes\):\s*\K[0-9]+' "$TIMELOG" | head -n 1 || echo 0)"
-  KV_EST_KB="$(CFG="$CFG" CL="$cl" KV_BYTES_PER_ELEM="${KV_BYTES_PER_ELEM:-2}" kv_est_kb || echo 0)"
+  KV_EST_KB="$(CFG="$CFG" CTX_LEN="$ctx_len" KV_BYTES_PER_ELEM="${KV_BYTES_PER_ELEM:-2}" kv_est_kb || echo 0)"
 
   echo "TTFT=$TTFT_MS ms  Prefill=$PREFILL_MS ms  Decode=$DECODE_MS ms  Decode/tok=$DECODE_PER_TOK ms  peakRSS=$PEAK_RSS_KB KB  KV_est=$KV_EST_KB KB"
 
-  echo "$TS,$GIT,$ARCH,tiny_llama,$mode,$cl,$pp,$tg,$THREADS,$TTFT_MS,$PREFILL_MS,$DECODE_MS,$DECODE_PER_TOK,$PEAK_RSS_KB,$KV_EST_KB" >> "$OUTCSV"
+  echo "$TS,$GIT,$ARCH,tiny_llama,$mode,$ctx_len,$pp,$tg,$THREADS,$TTFT_MS,$PREFILL_MS,$DECODE_MS,$DECODE_PER_TOK,$PEAK_RSS_KB,$KV_EST_KB" >> "$OUTCSV"
 }
 
-for CL in $CLS; do
-  run_one "decode_heavy" "$CL" "$TG_DH"
-  run_one "prefill_ttft" "$CL" "$TG_TTFT"
+for CTX in $CTX_LENS; do
+  run_one "decode_heavy" "$CTX" "$TG_DH"
+  run_one "prefill_ttft" "$CTX" "$TG_TTFT"
 done
 
 echo
