From b92cbed23d9f94369bedd2ed877752b5f045026f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 19 Jul 2026 15:39:08 +0800
Subject: [PATCH 1/5] feat: DRAM KV offloading via HiCache for all glm5.2 b300
 agentic points

Run 29651235293 showed the 1M-context corpus working set outgrowing the
HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned
at 1.0 while the radix hit rate collapsed from a ~0.97 theoretical
ceiling to 0.04-0.06, so every post-knee turn re-prefilled its whole
history and throughput fell together with interactivity, leaving the
frontier a single dominated cliff. Spill evicted prefixes to a per-rank
pinned host pool (0.80 x node DRAM / TP) instead of recomputing them,
following the qwen3.5 b300 hicache recipe; GLM-5.2 is plain GQA so there
is one host pool per rank and GB-based --hicache-size works (the DSv4
ratio-based sizing is a DSv4-only workaround).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../agentic/glm5.2_fp4_b300_sglang.sh         | 42 ++++++++++++++++---
 configs/nvidia-master.yaml                    |  4 +-
 perf-changelog.yaml                           |  8 ++++
 3 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
index 9f4b9e0871..246c1fab5a 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
@@ -13,16 +13,13 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION,
 #   EP_SIZE, DP_ATTENTION
+#
+# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-if [[ "$KV_OFFLOADING" != "none" ]]; then
-    echo "Error: KV_OFFLOADING=$KV_OFFLOADING is not supported by this recipe" >&2
-    exit 1
-fi
-
 if [[ -n "${SLURM_JOB_ID:-}" ]]; then
     echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
 fi
@@ -46,6 +43,40 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    # HiCache extends RadixAttention: prefixes evicted from the HBM KV pool
+    # spill to a pinned host pool instead of being recomputed. On the
+    # 1M-context agentic corpus the live working set outgrows HBM past
+    # conc 8 (TP8) / 64 (DP8) and the radix hit rate collapses to <0.1
+    # against a ~0.97 theoretical ceiling, so every turn re-prefills its
+    # whole history; the host tier restores those hits at C2C bandwidth.
+    # GLM-5.2 is plain GQA, so each rank owns a single host KV pool (no
+    # Mamba side pool as on Qwen3.5). SGLang --hicache-size is per rank,
+    # while the workflow input is a node-total DRAM budget: divide by TP,
+    # which equals the attention-DP rank count on the DP arm.
+    MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP))
+    HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
+    if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
+        echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-rank limit $MAX_HICACHE_SIZE_GB" >&2
+        exit 1
+    fi
+    if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
+        echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP" >&2
+        exit 1
+    fi
+    HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
+    echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank across TP=${TP}, write_policy=${HICACHE_WRITE_POLICY}"
+    CACHE_ARGS=(
+        --page-size 64
+        --enable-hierarchical-cache
+        --hicache-size "$HICACHE_SIZE_GB"
+        --hicache-io-backend kernel
+        --hicache-mem-layout page_first
+        --hicache-write-policy "$HICACHE_WRITE_POLICY"
+    )
+fi
+
 # With attention-DP, front the DP ranks with sglang-router using consistent
 # hashing on the AIPerf correlation id so multi-turn sessions stay on the DP
 # rank that holds their radix-cache prefix.
@@ -136,6 +167,7 @@ SGLANG_CMD=(
     --mem-fraction-static 0.85
     --max-running-requests "$MAX_RUNNING_REQUESTS"
     "${GRAPH_ARGS[@]}"
+    "${CACHE_ARGS[@]}"
     --watchdog-timeout 1800
     --enable-metrics
 )
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 4b35c30f87..3aac1b8378 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7873,5 +7873,5 @@ glm5.2-fp4-b300-sglang-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
-      - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 16, 32] }
+      - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } }
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 9e959c9464..f51f05f01e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4950,3 +4950,11 @@
   description:
     - "Bump vLLM image to v0.25.0"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256
+
+- config-keys:
+    - glm5.2-fp4-b300-sglang-agentic
+  description:
+    - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: per-rank pinned host pool = 0.80 x node DRAM / TP, page size 64, io-backend kernel, mem-layout page_first, write policy write_through_selective"
+    - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
+    - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
+  pr-link: XXX

From 9a8993157362022e63e107dc20534e5a7cb4e9d3 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 19 Jul 2026 15:41:45 +0800
Subject: [PATCH 2/5] chore: fill perf-changelog pr-link for glm5.2 b300
 hicache entry

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index f51f05f01e..d76c6bfa49 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4957,4 +4957,4 @@
     - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: per-rank pinned host pool = 0.80 x node DRAM / TP, page size 64, io-backend kernel, mem-layout page_first, write policy write_through_selective"
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
-  pr-link: XXX
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2279

From dc3a492b73191f8fd806407e672261cbda81c106 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 19 Jul 2026 16:03:41 +0800
Subject: [PATCH 3/5] fix: ratio-based HiCache sizing (0.75) for DSA-family
 GLM-5.2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Run 29678598595 OOM-killed both arms at scheduler init: GLM-5.2 runs
attention_backend=dsa (DeepSeek-sparse family; the 169.98 GB device KV
pool is replicated on every rank), so the GB-based --hicache-size of
TOTAL_CPU_DRAM_GB/TP pinned the whole 0.80-DRAM budget (8 x 299 GB =
2.4 TB) on top of 465 GB of weights on a 3 TB node. Control host
capacity through the host/device token-capacity ratio like the DSv4
recipe — but DSv4's ratio=2 default also OOMs here (2 x 170 GB x 8 =
2.7 TB; GLM-5.2's device pool is far larger than DSv4's), so use
fractional ratio 0.75 = ~128 GB/rank = ~1.0 TB total, matching the
cluster's proven host-pool envelope. Adopt DSv4's write_back / direct /
page_first_direct transfer flags. Validated on-node (b300-019,
2026-07-19): both sizing regressions reproduced, ratio 0.75 boots in
~290 s and survives a 4.2M-token overflow bench that forces eviction
through the DSA KV+INDEXER host pools.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .../agentic/glm5.2_fp4_b300_sglang.sh         | 41 +++++++++++--------
 perf-changelog.yaml                           |  2 +-
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
index 246c1fab5a..fcacde4d92 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
@@ -51,29 +51,34 @@ if require_agentic_kv_offload_backend hicache; then
     # conc 8 (TP8) / 64 (DP8) and the radix hit rate collapses to <0.1
     # against a ~0.97 theoretical ceiling, so every turn re-prefills its
     # whole history; the host tier restores those hits at C2C bandwidth.
-    # GLM-5.2 is plain GQA, so each rank owns a single host KV pool (no
-    # Mamba side pool as on Qwen3.5). SGLang --hicache-size is per rank,
-    # while the workflow input is a node-total DRAM budget: divide by TP,
-    # which equals the attention-DP rank count on the DP arm.
-    MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP))
-    HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
-    if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
-        echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-rank limit $MAX_HICACHE_SIZE_GB" >&2
+    # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every rank holds
+    # complete per-token KV (169.98 GB device pool per rank, replicated on
+    # all 8 ranks), so host capacity is controlled through the host/device
+    # token-capacity ratio like the DSv4 recipe, NOT a per-rank
+    # --hicache-size. A GB-based size of TOTAL_CPU_DRAM_GB/TP pinned the
+    # whole 0.80-DRAM budget (8 x 299 GB) at init on top of 465 GB of
+    # weights and OOM-killed the node (run 29678598595); DSv4's own
+    # ratio=2 default pins 2 x 170 GB x 8 = 2.7 TB here and OOMs too
+    # (GLM-5.2's device pool is far larger than DSv4's). Fractional 0.75
+    # = ~128 GB/rank = ~1.0 TB total, matching the cluster's proven ~1 TB
+    # host-pool envelope; validated on-node 2026-07-19 (boot + 4.2M-token
+    # overflow bench forcing eviction through the DSA KV+INDEXER pools).
+    DEFAULT_HICACHE_RATIO=0.75
+    HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}"
+    if awk -v r="$HICACHE_RATIO" -v cap="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(r > cap) }'; then
+        echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2
         exit 1
     fi
-    if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
-        echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP" >&2
-        exit 1
-    fi
-    HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
-    echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank across TP=${TP}, write_policy=${HICACHE_WRITE_POLICY}"
+    HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}"
+    HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
+    HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}"
+    echo "HiCache CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT"
     CACHE_ARGS=(
-        --page-size 64
         --enable-hierarchical-cache
-        --hicache-size "$HICACHE_SIZE_GB"
-        --hicache-io-backend kernel
-        --hicache-mem-layout page_first
+        --hicache-ratio "$HICACHE_RATIO"
         --hicache-write-policy "$HICACHE_WRITE_POLICY"
+        --hicache-io-backend "$HICACHE_IO_BACKEND"
+        --hicache-mem-layout "$HICACHE_MEM_LAYOUT"
     )
 fi
 
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index d76c6bfa49..e57215906e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4954,7 +4954,7 @@
 - config-keys:
     - glm5.2-fp4-b300-sglang-agentic
   description:
-    - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: per-rank pinned host pool = 0.80 x node DRAM / TP, page size 64, io-backend kernel, mem-layout page_first, write policy write_through_selective"
+    - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: DSv4-style host/device token-capacity ratio, tuned to 0.75 for GLM-5.2 (DSA/MLA-family: the 169.98 GB device pool is replicated on all 8 ranks, so ratio 0.75 = ~1.0 TB pinned host pools; ratio 2 and GB-based sizing both OOM-killed 3 TB nodes), io-backend direct, mem-layout page_first_direct, write policy write_back; validated on-node with an overflow bench forcing eviction through the DSA KV+INDEXER host pools"
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2279

From 1b64d4b0bc772a3b7e101ea6ce0d77a6d1bbf255 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 19 Jul 2026 23:00:29 +0800
Subject: [PATCH 4/5] chore: prepare PR 2279 ingest recovery


From bab978ec8c8ece5bf78ca04fa166c51f897c911b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 19 Jul 2026 23:01:52 +0800
Subject: [PATCH 5/5] fix: recover PR 2279 ingest

---
 perf-changelog.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e57215906e..005967fc36 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4958,3 +4958,11 @@
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2279
+
+- config-keys:
+    - glm5.2-fp4-b300-sglang-agentic
+  description:
+    - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: DSv4-style host/device token-capacity ratio, tuned to 0.75 for GLM-5.2 (DSA/MLA-family: the 169.98 GB device pool is replicated on all 8 ranks, so ratio 0.75 = ~1.0 TB pinned host pools; ratio 2 and GB-based sizing both OOM-killed 3 TB nodes), io-backend direct, mem-layout page_first_direct, write policy write_back; validated on-node with an overflow bench forcing eviction through the DSA KV+INDEXER host pools"
+    - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
+    - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280