From 2a04eb168633326fa2d0c02f7737c3b1b8cc2fd0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 20 Jul 2026 00:10:03 +0800
Subject: [PATCH 1/4] =?UTF-8?q?feat:=20DEP=20(EP8)=20for=20glm5.2=20b300?=
 =?UTF-8?q?=20sglang=20high-throughput=20arm,=20cap=20conc=20at=2064=20/?=
 =?UTF-8?q?=20=E4=B8=BA=20glm5.2=20b300=20sglang=20=E9=AB=98=E5=90=9E?=
 =?UTF-8?q?=E5=90=90=E5=88=86=E6=94=AF=E5=90=AF=E7=94=A8=20DEP(EP8)?=
 =?UTF-8?q?=E5=B9=B6=E5=B0=86=E5=B9=B6=E5=8F=91=E4=B8=8A=E9=99=90=E8=AE=BE?=
 =?UTF-8?q?=E4=B8=BA=2064?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add ep: 8 to the TP8/DP8 attention-DP arm of glm5.2-fp4-b300-sglang-agentic
so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8)
instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1).
Attention stays data-parallel behind sglang-router consistent hashing.
The recipe already passes --ep-size EP_SIZE unconditionally, so only the
master-config search space changes; no recipe logic change is needed.
Cap the DEP arm at conc 64 (conc-list [48, 64], was up to 512). The
low-latency TP8 arm (conc 1-32, fp8 KV + cutedsl GEMM) is unchanged. DEP
mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn arm.

中文：为 glm5.2-fp4-b300-sglang-agentic 的 TP8/DP8 注意力数据并行(attention-DP)
分支添加 ep: 8，使 混合专家(MoE) 层在全部 8 个 rank 上按 专家并行(EP) 运行
(--ep-size 8)，而非此前的 TP 切分(未设置 ep -> EP_SIZE=1)。注意力仍为数据
并行，并置于 sglang-router 一致性哈希之后以保持会话亲和性。该 recipe 脚本本就
无条件传入 --ep-size EP_SIZE，故仅需修改 master 配置的搜索空间，无需改动脚本
逻辑。将 DEP 分支并发上限设为 64(conc-list [48, 64]，原先最高 512)。低延迟
TP8 分支(并发 1-32，fp8 KV + cutedsl GEMM)保持不变。此 DEP 配置对齐已验证的
dsv4-fp4-b300-sglang tp8/ep8/dp-attn 分支。
---
 .../single_node/agentic/glm5.2_fp4_b300_sglang.sh      |  3 ++-
 configs/nvidia-master.yaml                             | 10 ++++++----
 perf-changelog.yaml                                    |  8 ++++++++
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
index fcacde4d92..6633236f9e 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
@@ -8,7 +8,8 @@ set -x
 # (https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.2), STP only:
 # the cookbook's EAGLE MTP variants are intentionally not wired up yet.
 #   DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM)
-#   DP_ATTENTION=true  -> high-throughput arm (TP8 + DP8 attention-DP)
+#   DP_ATTENTION=true  -> high-throughput DEP arm (TP8 + DP8 attention-DP +
+#                         EP_SIZE expert-parallel MoE via --ep-size)
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION,
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 3aac1b8378..8e0c55821e 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7858,9 +7858,10 @@ qwen3.5-fp8-gb300-dynamo-sglang:
 # single-node recipes (https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.2),
 # STP only (the cookbook's EAGLE MTP variants are deferred). The TP8 arm is the
 # cookbook low-latency recipe (fp8 KV, cutedsl bf16 GEMM) and covers the
-# interactivity end; the TP8/DP8 attention-DP arm is the cookbook
-# high-throughput recipe behind sglang-router consistent hashing for session
-# affinity. Conc lists are disjoint between arms so exp-names stay unique.
+# interactivity end; the TP8/DP8/EP8 DEP arm (attention-DP + expert-parallel
+# MoE) is the cookbook high-throughput recipe behind sglang-router consistent
+# hashing for session affinity. Conc lists are disjoint between arms so
+# exp-names stay unique.
 glm5.2-fp4-b300-sglang-agentic:
   image: lmsysorg/sglang:v0.5.15.post1-cu130
   model: nvidia/GLM-5.2-NVFP4
@@ -7874,4 +7875,5 @@ glm5.2-fp4-b300-sglang-agentic:
     - dram-utilization: 0.80
       search-space:
       - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 16, 32] }
-      - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } }
+      # DEP (attention-DP + EP8 expert-parallel MoE) throughput arm, capped at conc 64.
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], router: { name: sglang-router, version: "0.3.2" } }
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 005967fc36..d63a64a47c 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4966,3 +4966,11 @@
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280
+
+- config-keys:
+    - glm5.2-fp4-b300-sglang-agentic
+  description:
+    - "Convert the high-throughput arm to DEP: add ep: 8 to the TP8/DP8 attention-DP arm so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8) instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1). Attention stays data-parallel behind sglang-router consistent hashing; the recipe already passes --ep-size EP_SIZE unconditionally, so only the master-config search space changes"
+    - "Cap the DEP arm concurrency at 64: conc-list [48, 64] (was [48, 64, 96, 128, 192, 256, 512])"
+    - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING

From 6d088b087dbfd753a541a3e049c681a8a345300d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 20 Jul 2026 00:11:27 +0800
Subject: [PATCH 2/4] =?UTF-8?q?chore:=20fill=20perf-changelog=20pr-link=20?=
 =?UTF-8?q?for=20glm5.2=20b300=20DEP=20entry=20/=20=E8=A1=A5=E5=85=A8=20gl?=
 =?UTF-8?q?m5.2=20b300=20DEP=20=E6=9D=A1=E7=9B=AE=E7=9A=84=20perf-changelo?=
 =?UTF-8?q?g=20pr-link?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：将 glm5.2-fp4-b300-sglang-agentic DEP 条目的 pr-link 从占位符更新为 #2281。
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index d63a64a47c..6d602ce2b8 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4973,4 +4973,4 @@
     - "Convert the high-throughput arm to DEP: add ep: 8 to the TP8/DP8 attention-DP arm so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8) instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1). Attention stays data-parallel behind sglang-router consistent hashing; the recipe already passes --ep-size EP_SIZE unconditionally, so only the master-config search space changes"
     - "Cap the DEP arm concurrency at 64: conc-list [48, 64] (was [48, 64, 96, 128, 192, 256, 512])"
     - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281

From 0e363a69cea2df07a1f69fbacaeb08521ddbe0ac Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 20 Jul 2026 12:34:17 +0800
Subject: [PATCH 3/4] =?UTF-8?q?refactor:=20drop=20conc-512-tuned=20warmup?=
 =?UTF-8?q?=20grace=20override=20from=20glm5.2=20b300=20recipe=20/=20?=
 =?UTF-8?q?=E7=A7=BB=E9=99=A4=20glm5.2=20b300=20recipe=20=E4=B8=AD?=
 =?UTF-8?q?=E4=B8=BA=20conc=20512=20=E8=B0=83=E4=BC=98=E7=9A=84=20warmup?=
 =?UTF-8?q?=20grace=20=E8=A6=86=E7=9B=96=E5=80=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

AGENTIC_WARMUP_GRACE_PERIOD=3600 was tuned for the conc-512 DP8 point,
which this PR removes (DEP arm now caps at conc 64). At conc 64 the warmup
working set drains well within the default 1800s (warmup-grace-period is a
ceiling, not a fixed sleep), so fall back to the default and delete the
now-stale comment. No behavioral change at conc <= 64.

中文：AGENTIC_WARMUP_GRACE_PERIOD=3600 是为 conc 512 的 DP8 数据点调优的，而本
PR 已移除该点(DEP 分支并发上限现为 64)。在 conc 64 下，warmup 工作集会在默认
1800s 内完成 drain(warmup-grace-period 是上限而非固定等待),故回退到默认值并
删除已过时的注释。conc <= 64 下无行为变化。
---
 benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
index 6633236f9e..7fbea1cea1 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh
@@ -106,11 +106,6 @@ if [ "$DP_ATTENTION" = "true" ]; then
     # while KV usage sat at ~0.01). Use the cookbook's own dp8 lever from
     # the B200 cells (32768 = ~4096/rank).
     CHUNKED_PREFILL_SIZE=32768
-    # At conc 512 the saturation working set outlives the default 1800s
-    # warmup drain grace: the drain converges healthily (~0.45 req/s, zero
-    # errors) but needs ~2500s end to end. 3600 is a maximum wait, not a
-    # fixed sleep — lower-conc DPA points still finish as fast as they drain.
-    export AGENTIC_WARMUP_GRACE_PERIOD=3600
     PARALLEL_ARGS+=(
         --dp "$TP"
         --enable-dp-attention

From e47ca81dab2ab03fa8065e06fc8990ed5fcaf576 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 20 Jul 2026 19:07:32 +0800
Subject: [PATCH 4/4] =?UTF-8?q?feat:=20trim=20glm5.2=20b300=20DEP=20arm=20?=
 =?UTF-8?q?to=20conc=2048=20only=20/=20=E5=B0=86=20glm5.2=20b300=20DEP=20?=
 =?UTF-8?q?=E5=88=86=E6=94=AF=E6=94=B6=E6=95=9B=E4=B8=BA=E4=BB=85=20conc?=
 =?UTF-8?q?=2048?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The sweep (run 29717273540) showed DEP conc 48 is the frontier peak
(163 tok/s/gpu out, ttft 2.8s, GPU radix hit 0.93) while conc 64 overflows
the HBM+host radix cache (hit collapses to 0.57), thrashing on re-prefill
down to 41 tok/s/gpu at 56s ttft. conc 64 is strictly dominated by conc 48
on both throughput and interactivity and cannot be recovered without more
host-pool capacity than the 3TB node can give (ratio > 0.75 OOMs, per
#2279). Drop it: DEP arm conc-list is now [48]. The agentic eval also moves
to conc 48 (off the conc-64 point whose eval hung).

中文：扫描(run 29717273540)显示 DEP conc 48 是前沿峰值(输出 163 tok/s/gpu，
ttft 2.8s，GPU radix 命中 0.93)，而 conc 64 会使 HBM+host radix 缓存溢出(命中
率跌至 0.57)，陷入 re-prefill 抖动，降至 41 tok/s/gpu、ttft 56s。conc 64 在吞吐
与交互性两个维度上都被 conc 48 严格支配，且在 3TB 节点上无法通过增大 host pool
恢复(ratio > 0.75 会 OOM，见 #2279)。故移除：DEP 分支 conc-list 现为 [48]。
agentic 评估点也随之移到 conc 48(避开评估卡住的 conc 64 点)。
---
 configs/nvidia-master.yaml | 7 +++++--
 perf-changelog.yaml        | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 8e0c55821e..01ebad8b95 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7875,5 +7875,8 @@ glm5.2-fp4-b300-sglang-agentic:
     - dram-utilization: 0.80
       search-space:
       - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 16, 32] }
-      # DEP (attention-DP + EP8 expert-parallel MoE) throughput arm, capped at conc 64.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64], router: { name: sglang-router, version: "0.3.2" } }
+      # DEP (attention-DP + EP8 expert-parallel MoE) throughput arm at conc 48 — the
+      # frontier peak (163 tok/s/gpu out, ttft 2.8s). conc >=64 overflows the HBM+host
+      # radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is
+      # strictly dominated by conc 48 on both throughput and interactivity.
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48], router: { name: sglang-router, version: "0.3.2" } }
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 6d602ce2b8..e0afc35e88 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4971,6 +4971,6 @@
     - glm5.2-fp4-b300-sglang-agentic
   description:
     - "Convert the high-throughput arm to DEP: add ep: 8 to the TP8/DP8 attention-DP arm so the MoE layers run expert-parallel across all 8 ranks (--ep-size 8) instead of TP-sharded (the arm previously left ep unset -> EP_SIZE=1). Attention stays data-parallel behind sglang-router consistent hashing; the recipe already passes --ep-size EP_SIZE unconditionally, so only the master-config search space changes"
-    - "Cap the DEP arm concurrency at 64: conc-list [48, 64] (was [48, 64, 96, 128, 192, 256, 512])"
+    - "DEP arm runs conc-list [48] only (was [48, 64, 96, 128, 192, 256, 512]): conc 48 is the frontier peak (163 tok/s/gpu out, ttft 2.8s); conc >=64 overflows the HBM+host radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is strictly dominated"
     - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281