From 93e180d6b2c51dd536a34af951cbeabcdb0bf26b Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 13:10:27 -0400
Subject: [PATCH 01/42] feat(agentic): add GLM-5.2 MI325X full-context
 qualification
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a concurrency-1 TP8 SGLang AgentX configuration that explicitly allocates the native 1,048,576-token BF16 KV pool on MI325X and replays the unfiltered corpus.

中文：新增 GLM-5.2 MI325X TP8 SGLang AgentX 并发 1 验证配置，显式分配原生 1,048,576-token BF16 KV 缓存池，并回放未过滤的完整语料。
---
 .../single_node/agentic/glm5.2_fp8_mi325x.sh  | 100 ++++++++++++++++++
 configs/amd-master.yaml                       |  17 +++
 perf-changelog.yaml                           |   8 ++
 3 files changed, 125 insertions(+)
 create mode 100755 benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
new file mode 100755
index 0000000000..c3b9340c3b
--- /dev/null
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# Full-context AgentX qualification for GLM-5.2 FP8 on one 8xMI325X node.
+# The SGLang GLM-5.2 cookbook supports TP8 FP8 on MI325X with the DSA
+# TileLang backends. This recipe explicitly requests the native 1M context and
+# GPU-resident BF16 KV pool; SGLang startup fails if that allocation cannot be
+# satisfied. MTP is intentionally disabled because the AMD path is unvalidated.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
+
+if [[ "$TP" != "8" || "$EP_SIZE" != "1" || "$DP_ATTENTION" != "false" ]]; then
+    echo "Error: GLM-5.2 MI325X full-context qualification requires TP8/EP1 without DP attention" >&2
+    exit 1
+fi
+if [[ "$KV_OFFLOADING" != "none" ]]; then
+    echo "Error: KV_OFFLOADING=$KV_OFFLOADING is not supported by this recipe" >&2
+    exit 1
+fi
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then
+    export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
+fi
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+rocm-smi || true
+amd-smi || true
+
+# GLM-5.2 has a native 1M context, so replay the complete AgentX corpus rather
+# than the repository's default 256K-capped corpus for unrecognized families.
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
+resolve_trace_source
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+export PYTHONNOUSERSITE=1
+export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
+export SGLANG_TIMEOUT_KEEP_ALIVE=900
+
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS
+[ "$CUDA_GRAPH_MAX_BS" -gt 256 ] && CUDA_GRAPH_MAX_BS=256
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    --tp "$TP"
+    --ep-size "$EP_SIZE"
+    --dsa-prefill-backend tilelang
+    --dsa-decode-backend tilelang
+    --kv-cache-dtype bfloat16
+    --tool-call-parser glm47
+    --reasoning-parser glm45
+    --context-length 1048576
+    --max-total-tokens 1048576
+    --chunked-prefill-size 131072
+    --mem-fraction-static 0.85
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --watchdog-timeout 1800
+    --enable-metrics
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+
+echo "Starting SGLang server for MI325X..."
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+echo "Server PID: $SERVER_PID"
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [[ "${EVAL_ONLY}" == "true" ]]; then
+    export SWEBENCH_AGENT_STEP_LIMIT=150
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 7d8df0e03c..d2a5468a23 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2204,6 +2204,23 @@ minimaxm3-fp8-mi355x-vllm-disagg:
           dp-attn: false
           additional-settings:
           - "DECODE_NODES=1"
+
+# GLM-5.2 FP8 full-context AgentX qualification on one MI325X node. Keep this
+# first pass at concurrency 1: the purpose is to verify that TP8 can allocate
+# the explicit 1,048,576-token BF16 KV pool and replay the unfiltered corpus.
+glm5.2-fp8-mi325x-sglang-agentic:
+  image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x
+  model: zai-org/GLM-5.2-FP8
+  model-prefix: glm5.2
+  runner: cluster:mi325x-amds
+  precision: fp8
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - search-space:
+      - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1] }
+
 minimaxm3-fp8-mi300x-vllm-agentic:
   image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
   model: MiniMaxAI/MiniMax-M3-MXFP8
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index b358f10bf1..e9dcaa9322 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4933,3 +4933,11 @@
   description:
     - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182
+
+- config-keys:
+    - glm5.2-fp8-mi325x-sglang-agentic
+  description:
+    - "Add a GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context qualification"
+    - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
+    - "Use lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x with the AMD DSA TileLang prefill and decode backends; qualify concurrency 1 before expanding the frontier"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX

From 9cb86da2a547d32857779b13449defbdcbab6fa9 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 13:10:46 -0400
Subject: [PATCH 02/42] chore: link GLM-5.2 MI325X qualification PR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Record PR #2273 in the append-only benchmark changelog entry.

中文：在仅追加的基准测试变更日志条目中记录 PR #2273。
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e9dcaa9322..3ba475d47d 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4940,4 +4940,4 @@
     - "Add a GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context qualification"
     - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
     - "Use lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x with the AMD DSA TileLang prefill and decode backends; qualify concurrency 1 before expanding the frontier"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273

From 5a80bfd389a11b1efa130a2a305db5ed7146a8a3 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 13:26:34 -0400
Subject: [PATCH 03/42] fix(agentic): use portable DSA top-k on MI325X
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid the gfx942 sgl-kernel JIT failure caused by its CUDA cooperative_groups.h include by selecting the Torch DSA top-k fallback and disabling fused top-k.

中文：MI30X 镜像的 sgl-kernel JIT 在 gfx942 编译时错误引用 CUDA cooperative_groups.h；改用 Torch DSA top-k 回退实现并关闭融合 top-k，避免启动失败。
---
 benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index c3b9340c3b..e1d0a42947 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -51,6 +51,10 @@ mkdir -p "$RESULT_DIR"
 export PYTHONNOUSERSITE=1
 export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 export SGLANG_TIMEOUT_KEEP_ALIVE=900
+# The mi30x image's sgl-kernel DSA top-k JIT includes CUDA's
+# cooperative_groups.h while compiling for gfx942. Use SGLang's portable
+# Torch fallback and disable the fused top-k path that requires that kernel.
+export SGLANG_DSA_FUSE_TOPK=false
 
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS
@@ -67,6 +71,7 @@ SGLANG_CMD=(
     --ep-size "$EP_SIZE"
     --dsa-prefill-backend tilelang
     --dsa-decode-backend tilelang
+    --dsa-topk-backend torch
     --kv-cache-dtype bfloat16
     --tool-call-parser glm47
     --reasoning-parser glm45

From 884dc14dcc886425c92674460ce14cefcb0a5f04 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 13:33:12 -0400
Subject: [PATCH 04/42] fix(agentic): disable CUDA-only top-k planning on
 MI325X
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Disable the independently gated topk_v2 CUDA-graph planner so the configured Torch DSA top-k fallback is honored on gfx942.

中文：在 MI325X 上禁用独立控制的 topk_v2 CUDA Graph 规划内核，确保 gfx942 使用已配置的 Torch DSA top-k 回退路径。
---
 benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index e1d0a42947..cc9cc31192 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -53,8 +53,10 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 export SGLANG_TIMEOUT_KEEP_ALIVE=900
 # The mi30x image's sgl-kernel DSA top-k JIT includes CUDA's
 # cooperative_groups.h while compiling for gfx942. Use SGLang's portable
-# Torch fallback and disable the fused top-k path that requires that kernel.
+# Torch fallback and disable both the fused top-k path and its independently
+# gated CUDA-graph planning kernel.
 export SGLANG_DSA_FUSE_TOPK=false
+export SGLANG_OPT_USE_TOPK_V2=false
 
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS

From c133f24c9670336e94fbe665312fdd8936f5001d Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 15:24:18 -0400
Subject: [PATCH 05/42] feat(agentic): expand GLM-5.2 MI325X curve
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Expand the full-context AgentX configuration to sample concurrency densely around the expected BF16 KV-pool capacity knee.\n\n中文：扩展 GLM-5.2 MI325X 全上下文 AgentX 配置，在预期的 BF16 KV 缓存池容量拐点附近进行密集并发采样。
---
 configs/amd-master.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index d2a5468a23..587b035cc6 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2205,9 +2205,9 @@ minimaxm3-fp8-mi355x-vllm-disagg:
           additional-settings:
           - "DECODE_NODES=1"
 
-# GLM-5.2 FP8 full-context AgentX qualification on one MI325X node. Keep this
-# first pass at concurrency 1: the purpose is to verify that TP8 can allocate
-# the explicit 1,048,576-token BF16 KV pool and replay the unfiltered corpus.
+# GLM-5.2 FP8 full-context AgentX curve on one MI325X node. The qualification
+# run used about 35% of the explicit 1,048,576-token BF16 KV pool at concurrency
+# 1, so sample densely around the expected concurrency 3-4 capacity knee.
 glm5.2-fp8-mi325x-sglang-agentic:
   image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x
   model: zai-org/GLM-5.2-FP8
@@ -2219,7 +2219,7 @@ glm5.2-fp8-mi325x-sglang-agentic:
   scenarios:
     agentic-coding:
     - search-space:
-      - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1] }
+      - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8] }
 
 minimaxm3-fp8-mi300x-vllm-agentic:
   image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914

From 02d18049109a8d8023cc293929a4aab415ffe44c Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sat, 18 Jul 2026 15:31:19 -0400
Subject: [PATCH 06/42] feat(agentic): add GLM-5.2 MI325X disagg sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add full-context AgentX 1P1D, 1P2D, and 2P1D TP8 topology arms, plus MI325X Slurm launch and artifact plumbing. Keep EP/DP attention disabled on the Broadcom bnxt_re fleet.

中文：新增 GLM-5.2 MI325X 全上下文 AgentX 分离式推理扫描，覆盖 TP8 的 1P1D、1P2D 和 2P1D 拓扑，并补齐 MI325X Slurm 启动与产物收集流程。Broadcom bnxt_re 集群暂不启用 EP/DP attention。
---
 benchmarks/multi_node/amd_utils/env.sh        |   8 +-
 benchmarks/multi_node/amd_utils/job.slurm     |  41 ++++-
 benchmarks/multi_node/amd_utils/models.yaml   |  32 ++++
 .../multi_node/amd_utils/server_sglang.sh     |  57 ++++--
 benchmarks/multi_node/amd_utils/submit.sh     |  22 ++-
 .../glm5.2_fp8_mi325x_sglang-disagg.sh        |  47 +++++
 configs/amd-master.yaml                       |  79 +++++++++
 perf-changelog.yaml                           |   9 +
 runners/launch_mi325x-amds.sh                 |   4 +
 runners/launch_mi325x-amds_multinode.sh       | 162 ++++++++++++++++++
 10 files changed, 444 insertions(+), 17 deletions(-)
 create mode 100755 benchmarks/multi_node/glm5.2_fp8_mi325x_sglang-disagg.sh
 create mode 100755 runners/launch_mi325x-amds_multinode.sh

diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh
index c75675cb1c..5932695489 100755
--- a/benchmarks/multi_node/amd_utils/env.sh
+++ b/benchmarks/multi_node/amd_utils/env.sh
@@ -206,12 +206,18 @@ else
     export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
 
     # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading
-    if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then
+    MODEL_CONFIG_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
+    if [[ "$MODEL_CONFIG_KEY" == "GLM-5-FP8" || "$MODEL_CONFIG_KEY" == "GLM-5.2-FP8" ]]; then
         export SGLANG_ROCM_FUSED_DECODE_MLA=0
         export ROCM_QUICK_REDUCE_QUANTIZATION=INT4
         export SAFETENSORS_FAST_GPU=1
     fi
 
+    if [[ "$MODEL_CONFIG_KEY" == "GLM-5.2-FP8" ]]; then
+        export SGLANG_DSA_FUSE_TOPK=false
+        export SGLANG_OPT_USE_TOPK_V2=false
+    fi
+
     # Disable allocating memory in one pass
     export MORI_SHMEM_MODE=ISOLATION
 
diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 4638e5cadb..85c014fa1a 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -42,13 +42,14 @@ if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then
 fi
 
 MODEL_NAME="${MODEL_NAME:-None}"
-if ! grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then
-    echo "Error: Model '$MODEL_NAME' not found in $MODELS_YAML"
+MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
+if ! grep -Fqx "${MODEL_YAML_KEY}:" "$MODELS_YAML"; then
+    echo "Error: Model '$MODEL_YAML_KEY' not found in $MODELS_YAML"
     echo "Available models:"
     grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/  - /'
     exit 1
 fi
-echo "Model found: $MODEL_NAME"
+echo "Model configuration found: $MODEL_YAML_KEY"
 
 RUN_FILE="server.sh"
 echo "Runfile set: $RUN_FILE"
@@ -262,6 +263,7 @@ export MODEL_DIR=$MODEL_DIR
 export xP=$xP
 export yD=$yD
 export MODEL_NAME=$MODEL_NAME
+export MODEL_YAML_KEY=$MODEL_YAML_KEY
 export USER_NAME=$USER_NAME
 export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')"
 export GPUS_PER_NODE=$GPUS_PER_NODE
@@ -287,9 +289,24 @@ export RUNNER_TYPE="${RUNNER_TYPE:-}"
 export RESULT_FILENAME="${RESULT_FILENAME:-}"
 export SPEC_DECODING="${SPEC_DECODING:-}"
 export IS_MULTINODE="${IS_MULTINODE:-false}"
+export SCENARIO_TYPE="${SCENARIO_TYPE:-fixed-seq-len}"
+export IS_AGENTIC="${IS_AGENTIC:-0}"
+export CONC="${CONC:-}"
+export CONC_LIST="${CONC_LIST:-}"
+export DURATION="${DURATION:-3600}"
+export KV_OFFLOADING="${KV_OFFLOADING:-none}"
+export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}"
+export KV_OFFLOAD_BACKEND_METADATA="${KV_OFFLOAD_BACKEND_METADATA:-}"
+export ROUTER_METADATA="${ROUTER_METADATA:-}"
+export KV_P2P_TRANSFER="${KV_P2P_TRANSFER:-}"
+export TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-0}"
+export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
+export MODEL="${MODEL:-}"
+export WEKA_LOADER_OVERRIDE="${WEKA_LOADER_OVERRIDE:-}"
 
 SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_')
-export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}"
+SANITIZED_MODEL=$(echo "$MODEL_NAME" | tr -c 'a-zA-Z0-9_.-' '_')
+export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${SANITIZED_MODEL}_${SLURM_JOB_ID}"
 
 # vLLM external router container.
 # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older
@@ -339,6 +356,7 @@ DOCKER_ENV_COMMON=(
     -e NODE0_ADDR=\$NODE0_ADDR
     -e MODEL_DIR=/models
     -e MODEL_NAME=\$MODEL_NAME
+    -e MODEL_YAML_KEY=\$MODEL_YAML_KEY
     -e GPUS_PER_NODE=\$GPUS_PER_NODE
     -e xP=\$xP
     -e yD=\$yD
@@ -371,6 +389,21 @@ DOCKER_ENV_COMMON=(
     -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP
     -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE
     -e IS_MULTINODE=\$IS_MULTINODE
+    -e SCENARIO_TYPE=\$SCENARIO_TYPE
+    -e IS_AGENTIC=\$IS_AGENTIC
+    -e CONC=\$CONC
+    -e "CONC_LIST=\$CONC_LIST"
+    -e DURATION=\$DURATION
+    -e KV_OFFLOADING=\$KV_OFFLOADING
+    -e KV_OFFLOAD_BACKEND=\$KV_OFFLOAD_BACKEND
+    -e "KV_OFFLOAD_BACKEND_METADATA=\$KV_OFFLOAD_BACKEND_METADATA"
+    -e "ROUTER_METADATA=\$ROUTER_METADATA"
+    -e KV_P2P_TRANSFER=\$KV_P2P_TRANSFER
+    -e TOTAL_CPU_DRAM_GB=\$TOTAL_CPU_DRAM_GB
+    -e MAX_MODEL_LEN=\$MAX_MODEL_LEN
+    -e MODEL=\$MODEL
+    -e WEKA_LOADER_OVERRIDE=\$WEKA_LOADER_OVERRIDE
+    -e HF_HUB_CACHE=/models
 )
 
 # Engine-specific env vars
diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml
index bc2b39aa07..47c36f4d9b 100644
--- a/benchmarks/multi_node/amd_utils/models.yaml
+++ b/benchmarks/multi_node/amd_utils/models.yaml
@@ -264,6 +264,38 @@ GLM-5-FP8:
       chunked_prefill_size: 262144
       cuda_graph_bs_range: "1-128"
 
+GLM-5.2-FP8:
+  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --dsa-topk-backend torch --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
+  mtp_flags: ""
+  dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
+  ep_flags: "--moe-a2a-backend mori"
+  prefill:
+    mem_fraction_static: 0.85
+    disable_radix_cache: false
+    dp:
+      max_running_requests: 16
+      chunked_prefill_size: 131072
+      cuda_graph_bs: "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16"
+    no_dp:
+      max_running_requests: 16
+      chunked_prefill_size: 131072
+      cuda_graph_bs_range: "1-16"
+  decode:
+    mem_fraction_static: 0.85
+    prefill_round_robin_balance: true
+    dp:
+      max_running_requests: 16
+      chunked_prefill_size: 131072
+      cuda_graph_bs_range: "1-16"
+    ep_only:
+      max_running_requests: 16
+      chunked_prefill_size: 131072
+      cuda_graph_bs_range: "1-16"
+    no_dp:
+      max_running_requests: 16
+      chunked_prefill_size: 131072
+      cuda_graph_bs_range: "1-16"
+
 DeepSeek-R1-0528-MXFP4-Preview:
   base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori"
   mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1"
diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index ec5805eb0e..529eb8b900 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -10,6 +10,7 @@ NODE0_ADDR="${NODE0_ADDR:-localhost}"
 NODE_RANK="${NODE_RANK:-0}"
 MODEL_DIR="${MODEL_DIR:-}"
 MODEL_NAME="${MODEL_NAME:-}"
+MODEL_CONFIG_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
 
 xP="${xP:-1}" #-> Number of Prefill Workers
 yD="${yD:-1}" #-> Number of Decode Workers
@@ -80,7 +81,7 @@ eval "$(python3 -c "
 import yaml, sys, os
 
 config_path = '${MODELS_YAML}'
-model_name = '${MODEL_NAME}'
+model_name = '${MODEL_CONFIG_KEY}'
 
 with open(config_path) as f:
     models = yaml.safe_load(f)
@@ -173,7 +174,7 @@ print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"')
 print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"')
 ")"
 
-echo "Loaded model configuration for: $MODEL_NAME"
+echo "Loaded model configuration for: $MODEL_CONFIG_KEY"
 
 # Compute DP-dependent prefill parameters
 if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then
@@ -379,8 +380,8 @@ build_server_config() {
 PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE")
 DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE")
 
-if [[ -n "$MODEL_NAME" ]]; then
-    echo "Using model-specific configuration for: $MODEL_NAME"
+if [[ -n "$MODEL_CONFIG_KEY" ]]; then
+    echo "Using model-specific configuration for: $MODEL_CONFIG_KEY"
 fi
 
 if [[ "${EVAL_ONLY:-false}" == "true" ]] || [[ "${RUN_EVAL:-false}" == "true" ]]; then
@@ -506,12 +507,16 @@ if [ "$NODE_RANK" -eq 0 ]; then
     fi
     echo "Congratulations!!! All prefill and decode servers are up . . ."
 
+    ROUTER_POLICY="random"
+    if [[ "${IS_AGENTIC:-0}" == "1" ]]; then
+        ROUTER_POLICY="cache_aware"
+    fi
     ROUTER_CMD="python -m sglang_router.launch_router \
         --pd-disaggregation \
         --port 30000 \
-        --policy random \
+        --policy ${ROUTER_POLICY} \
         --prefill-policy random \
-        --decode-policy random \
+        --decode-policy ${ROUTER_POLICY} \
         ${PREFILL_ARGS} \
         ${DECODE_ARGS}"
 
@@ -573,11 +578,41 @@ if [ "$NODE_RANK" -eq 0 ]; then
         export IS_MTP=false
     fi
 
-    # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier
-    BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \
-        $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \
-        ${BENCH_OUTPUT_LEN} "${BENCH_MAX_CONCURRENCY}" ${BENCH_REQUEST_RATE} \
-        ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}"
+    if [[ "${IS_AGENTIC:-0}" == "1" ]]; then
+        export PORT=30000
+        export INFMAX_CONTAINER_WORKSPACE=/workspace
+        export RESULT_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/agentic"
+        export AGENTIC_OUTPUT_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/agentic-output"
+        export CONC_LIST="${BENCH_MAX_CONCURRENCY//x/ }"
+        export CONC="${CONC:-${CONC_LIST%% *}}"
+        export PREFILL_TP="$PREFILL_TP_SIZE"
+        export PREFILL_EP=1
+        [[ "$PREFILL_ENABLE_EP" == "true" ]] && export PREFILL_EP="$PREFILL_TP_SIZE"
+        export PREFILL_DP_ATTENTION="$PREFILL_ENABLE_DP"
+        export PREFILL_NUM_WORKERS="$xP"
+        export DECODE_TP="$DECODE_TP_SIZE"
+        export DECODE_EP=1
+        [[ "$DECODE_ENABLE_EP" == "true" ]] && export DECODE_EP="$DECODE_TP_SIZE"
+        export DECODE_DP_ATTENTION="$DECODE_ENABLE_DP"
+        export DECODE_NUM_WORKERS="$yD"
+
+        metrics_urls=()
+        for i in $(seq 0 $((xP - 1))); do
+            metrics_urls+=("http://${IP_ARRAY[$((i * PREFILL_NODES_PER_WORKER))]}:8000/metrics")
+        done
+        for i in $(seq 0 $((yD - 1))); do
+            metrics_urls+=("http://${IP_ARRAY[$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET))]}:8000/metrics")
+        done
+        export AIPERF_SERVER_METRICS_URLS
+        AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${metrics_urls[*]}")
+        BENCH_CMD="bash /workspace/benchmarks/multi_node/agentic_srt.sh"
+    else
+        # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier
+        BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \
+            $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \
+            ${BENCH_OUTPUT_LEN} "${BENCH_MAX_CONCURRENCY}" ${BENCH_REQUEST_RATE} \
+            ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}"
+    fi
 
     if [[ "${EVAL_ONLY:-false}" == "true" ]]; then
         echo "EVAL_ONLY mode: skipping throughput benchmark"
diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh
index 262fd25aa0..99f147f91d 100755
--- a/benchmarks/multi_node/amd_utils/submit.sh
+++ b/benchmarks/multi_node/amd_utils/submit.sh
@@ -121,6 +121,7 @@ export DECODE_MTP_SIZE=${DECODE_MTP_SIZE}
 export NUM_NODES=$NUM_NODES
 export GPUS_PER_NODE=$GPUS_PER_NODE
 export MODEL_NAME=$MODEL_NAME
+export MODEL_YAML_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
 export BENCH_INPUT_LEN=${ISL}
 export BENCH_OUTPUT_LEN=${OSL}
 export BENCH_NUM_PROMPTS_MULTIPLIER=${BENCH_NUM_PROMPTS_MULTIPLIER:-10}
@@ -145,6 +146,20 @@ export RUNNER_TYPE="${RUNNER_TYPE:-}"
 export RESULT_FILENAME="${RESULT_FILENAME:-}"
 export SPEC_DECODING="${SPEC_DECODING:-}"
 export IS_MULTINODE="${IS_MULTINODE:-false}"
+export SCENARIO_TYPE="${SCENARIO_TYPE:-fixed-seq-len}"
+export IS_AGENTIC="${IS_AGENTIC:-0}"
+export CONC="${CONC:-${CONCURRENCIES%%x*}}"
+export CONC_LIST="${CONC_LIST:-${CONCURRENCIES//x/ }}"
+export DURATION="${DURATION:-3600}"
+export KV_OFFLOADING="${KV_OFFLOADING:-none}"
+export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}"
+export KV_OFFLOAD_BACKEND_METADATA="${KV_OFFLOAD_BACKEND_METADATA:-}"
+export ROUTER_METADATA="${ROUTER_METADATA:-}"
+export KV_P2P_TRANSFER="${KV_P2P_TRANSFER:-}"
+export TOTAL_CPU_DRAM_GB="${TOTAL_CPU_DRAM_GB:-0}"
+export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
+export MODEL="${MODEL:-}"
+export WEKA_LOADER_OVERRIDE="${WEKA_LOADER_OVERRIDE:-}"
 
 # Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output.
 export BENCHMARK_LOGS_DIR="${BENCHMARK_LOGS_DIR:-$(pwd)/benchmark_logs}"
@@ -166,7 +181,12 @@ fi
 # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets).
 # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames.
 EXCLUDE_OPT=()
-SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g11,mia1-p01-g12,mia1-p01-g15}"
+if [[ -z "${SLURM_EXCLUDE_NODES+x}" ]]; then
+    case "$(hostname -s)" in
+        mia1*) SLURM_EXCLUDE_NODES="mia1-p01-g11,mia1-p01-g12,mia1-p01-g15" ;;
+        *) SLURM_EXCLUDE_NODES="" ;;
+    esac
+fi
 if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then
     EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES")
 fi
diff --git a/benchmarks/multi_node/glm5.2_fp8_mi325x_sglang-disagg.sh b/benchmarks/multi_node/glm5.2_fp8_mi325x_sglang-disagg.sh
new file mode 100755
index 0000000000..729cf01423
--- /dev/null
+++ b/benchmarks/multi_node/glm5.2_fp8_mi325x_sglang-disagg.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source "$(dirname "$0")/../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    SPEC_DECODING \
+    MODEL_PATH \
+    MODEL_NAME \
+    PREFILL_NUM_WORKERS \
+    PREFILL_TP \
+    PREFILL_EP \
+    PREFILL_DP_ATTN \
+    DECODE_NUM_WORKERS \
+    DECODE_TP \
+    DECODE_EP \
+    DECODE_DP_ATTN \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO \
+    FRAMEWORK
+
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils"
+
+export TIME_LIMIT="08:00:00"
+export CONTAINER_IMAGE="$IMAGE"
+export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}"
+export MAX_MODEL_LEN=1048576
+export WEKA_LOADER_OVERRIDE="${WEKA_LOADER_OVERRIDE:-semianalysis_cc_traces_weka_062126}"
+
+[[ "$PREFILL_EP" -eq 1 ]] && export PREFILL_ENABLE_EP=false || export PREFILL_ENABLE_EP=true
+[[ "$PREFILL_DP_ATTN" == "true" ]] && export PREFILL_ENABLE_DP=true || export PREFILL_ENABLE_DP=false
+[[ "$DECODE_EP" -eq 1 ]] && export DECODE_ENABLE_EP=false || export DECODE_ENABLE_EP=true
+[[ "$DECODE_DP_ATTN" == "true" ]] && export DECODE_ENABLE_DP=true || export DECODE_ENABLE_DP=false
+
+bash ./submit.sh \
+    "$PREFILL_NODES" "$PREFILL_NUM_WORKERS" \
+    "$DECODE_NODES" "$DECODE_NUM_WORKERS" \
+    "$ISL" "$OSL" "${CONC_LIST// /x}" inf \
+    "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" \
+    "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" \
+    "$PREFILL_TP" "$DECODE_TP" \
+    "$RANDOM_RANGE_RATIO" "${NODELIST:-}"
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 587b035cc6..dff6539912 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2221,6 +2221,85 @@ glm5.2-fp8-mi325x-sglang-agentic:
     - search-space:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8] }
 
+# GLM-5.2 FP8 full-context AgentX disaggregation on MI325X. Follow the
+# DeepSeek-V4-Pro AgentX convention: keep each engine/topology to at most four
+# sequential concurrency points and use separate topology arms. MI325X's
+# Broadcom bnxt_re MoRI all-to-all path is still unstable, so this sweep varies
+# TP8 prefill/decode worker ratios without EP or DP attention.
+glm5.2-fp8-mi325x-sglang-disagg-agentic:
+  image: semianalysiswork/sgl-bnxt-cdna3:pr22665-bnxt
+  model: zai-org/GLM-5.2-FP8
+  model-prefix: glm5.2
+  runner: cluster:mi325x-amds
+  precision: fp8
+  framework: sglang-disagg
+  router: { name: sglang-router, version: "0.3.2" }
+  kv-p2p-transfer: mori
+  multinode: true
+  disagg: true
+  scenarios:
+    agentic-coding:
+    - search-space:
+      # Balanced baseline: one TP8 prefill worker and one TP8 decode worker.
+      - spec-decoding: none
+        conc-list: [1, 2, 3, 4]
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=1"
+          - "MODEL_YAML_KEY=GLM-5.2-FP8"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=1"
+          - "DECODE_MTP_SIZE=0"
+
+      # Decode-heavy: split conversations across two independent TP8 decoders.
+      - spec-decoding: none
+        conc-list: [3, 4, 6, 8]
+        prefill:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=1"
+          - "MODEL_YAML_KEY=GLM-5.2-FP8"
+        decode:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=2"
+          - "DECODE_MTP_SIZE=0"
+
+      # Prefill-heavy: two TP8 prefill workers feeding one TP8 decoder.
+      - spec-decoding: none
+        conc-list: [2, 3, 4, 6]
+        prefill:
+          num-worker: 2
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "PREFILL_NODES=2"
+          - "MODEL_YAML_KEY=GLM-5.2-FP8"
+        decode:
+          num-worker: 1
+          tp: 8
+          ep: 1
+          dp-attn: false
+          additional-settings:
+          - "DECODE_NODES=1"
+          - "DECODE_MTP_SIZE=0"
+
 minimaxm3-fp8-mi300x-vllm-agentic:
   image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
   model: MiniMaxAI/MiniMax-M3-MXFP8
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 3ba475d47d..0088b110c8 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4941,3 +4941,12 @@
     - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
     - "Use lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x with the AMD DSA TileLang prefill and decode backends; qualify concurrency 1 before expanding the frontier"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273
+
+- config-keys:
+    - glm5.2-fp8-mi325x-sglang-agentic
+    - glm5.2-fp8-mi325x-sglang-disagg-agentic
+  description:
+    - "Expand the single-node GLM-5.2 MI325X full-context AgentX curve to concurrency [1, 2, 3, 4, 5, 6, 8] around the measured KV-capacity knee"
+    - "Add TP8 SGLang disaggregated AgentX sweeps for balanced 1P1D, decode-heavy 1P2D, and prefill-heavy 2P1D topologies, with at most four concurrency points per server allocation"
+    - "Keep EP and DP attention disabled because MoRI all-to-all remains unstable on the MI325X Broadcom bnxt_re fleet"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273
diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh
index a2cad5d496..663dbd199a 100644
--- a/runners/launch_mi325x-amds.sh
+++ b/runners/launch_mi325x-amds.sh
@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+if [[ "${IS_MULTINODE:-false}" == "true" ]]; then
+    exec bash "$(dirname "$0")/launch_mi325x-amds_multinode.sh"
+fi
+
 export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/"
 
 PARTITION="compute"
diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
new file mode 100755
index 0000000000..cbeebbc9b4
--- /dev/null
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+scancel_sync() {
+    local job_id="$1"
+    local deadline=$((SECONDS + 600))
+    scancel "$job_id" 2>/dev/null || true
+    while squeue -h -j "$job_id" 2>/dev/null | grep -q .; do
+        if (( SECONDS >= deadline )); then
+            echo "WARNING: Slurm job $job_id still exists after 600 seconds" >&2
+            return 1
+        fi
+        sleep 10
+    done
+}
+
+stage_server_logs() {
+    [[ -d "${BENCHMARK_LOGS_DIR:-}" ]] || return 0
+    local archive
+    archive=$(mktemp /tmp/inferencex-mi325x-server-logs.XXXXXX.tar.gz)
+    sudo tar -czf "$archive" \
+        --exclude='*/agentic' \
+        --exclude='*/agentic/*' \
+        --exclude='*/agentic-output' \
+        --exclude='*/agentic-output/*' \
+        -C "$BENCHMARK_LOGS_DIR" . 2>/dev/null || true
+    sudo chown "$USER" "$archive" 2>/dev/null || true
+    if [[ -s "$archive" ]]; then
+        mv -f "$archive" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"
+    else
+        rm -f "$archive"
+    fi
+}
+
+cleanup() {
+    local rc=$?
+    trap - EXIT
+    if [[ -n "${JOB_ID:-}" ]] && squeue -h -j "$JOB_ID" 2>/dev/null | grep -q .; then
+        scancel_sync "$JOB_ID" || true
+    fi
+    stage_server_logs || true
+    if [[ -n "${BENCHMARK_LOGS_DIR:-}" && -d "$BENCHMARK_LOGS_DIR" ]]; then
+        sudo rm -rf -- "$BENCHMARK_LOGS_DIR" 2>/dev/null || true
+    fi
+    if [[ -n "${BENCHMARK_LOGS_PARENT:-}" && -d "$BENCHMARK_LOGS_PARENT" ]]; then
+        rmdir "$BENCHMARK_LOGS_PARENT" 2>/dev/null || true
+    fi
+    exit "$rc"
+}
+trap cleanup EXIT
+
+export SLURM_ACCOUNT="$USER"
+export SLURM_PARTITION="compute"
+export SLURM_JOB_NAME="benchmark-sglang-disagg.job"
+export HF_HUB_CACHE_MOUNT="/nfsdata/sa/gharunner/gharunners/hf-hub-cache"
+export MODEL_PATH="$HF_HUB_CACHE_MOUNT"
+export MODEL_YAML_KEY="${MODEL_YAML_KEY:-${MODEL##*/}}"
+export GPUS_PER_NODE=8
+export IBDEVICES="${IBDEVICES:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8}"
+export MORI_RDMA_TC="${MORI_RDMA_TC:-104}"
+
+if [[ -z "${MODEL_NAME:-}" ]]; then
+    hf_dir="models--${MODEL//\//--}"
+    snapshot_root="$MODEL_PATH/$hf_dir/snapshots"
+    if [[ -d "$snapshot_root" ]]; then
+        snapshot=$(find "$snapshot_root" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort | tail -1)
+    else
+        snapshot=""
+    fi
+    if [[ -n "$snapshot" ]]; then
+        export MODEL_NAME="$hf_dir/snapshots/$snapshot"
+    elif [[ -d "$MODEL_PATH/${MODEL##*/}" ]]; then
+        export MODEL_NAME="${MODEL##*/}"
+    else
+        echo "ERROR: model '$MODEL' is not staged under $MODEL_PATH" >&2
+        exit 1
+    fi
+fi
+
+runner_tag=${RUNNER_NAME//[^a-zA-Z0-9_.-]/_}
+BENCHMARK_LOGS_PARENT="$(dirname "$GITHUB_WORKSPACE")/.inferencex-benchmark-logs"
+mkdir -p "$BENCHMARK_LOGS_PARENT"
+BENCHMARK_LOGS_DIR=$(mktemp -d "$BENCHMARK_LOGS_PARENT/mi325x-${runner_tag}.XXXXXX")
+export BENCHMARK_LOGS_PARENT
+export BENCHMARK_LOGS_DIR
+
+script_name="${EXP_NAME%%_*}_${PRECISION}_mi325x_${FRAMEWORK}.sh"
+benchmark_script="benchmarks/multi_node/$script_name"
+if [[ ! -f "$benchmark_script" ]]; then
+    echo "ERROR: $benchmark_script does not exist" >&2
+    exit 1
+fi
+
+echo "Submitting $benchmark_script with model path $MODEL_NAME"
+JOB_ID=$(bash "$benchmark_script" | tail -1)
+if ! [[ "$JOB_ID" =~ ^[0-9]+$ ]]; then
+    echo "ERROR: benchmark submission returned invalid job ID '$JOB_ID'" >&2
+    exit 1
+fi
+
+log_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out"
+while [[ ! -f "$log_file" ]]; do
+    if ! squeue -h -j "$JOB_ID" 2>/dev/null | grep -q .; then
+        echo "ERROR: Slurm job $JOB_ID ended before creating its output log" >&2
+        scontrol show job "$JOB_ID" 2>/dev/null || true
+        exit 1
+    fi
+    sleep 5
+done
+
+(
+    while squeue -h -j "$JOB_ID" 2>/dev/null | grep -q .; do
+        sleep 10
+    done
+) &
+poll_pid=$!
+tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null || true
+wait "$poll_pid"
+
+job_state=$(sacct -n -X -j "$JOB_ID" --format=State --parsable2 2>/dev/null | head -1 | cut -d'|' -f1)
+job_exit=$(sacct -n -X -j "$JOB_ID" --format=ExitCode --parsable2 2>/dev/null | head -1 | cut -d'|' -f1)
+echo "Slurm job $JOB_ID finished: state=${job_state:-unknown}, exit=${job_exit:-unknown}"
+
+if [[ "${IS_AGENTIC:-0}" == "1" ]]; then
+    mapfile -d '' aggregate_files < <(
+        find "$BENCHMARK_LOGS_DIR/logs" -type f \
+            -path '*/agentic-output/*.json' -print0 2>/dev/null
+    )
+    for result_file in "${aggregate_files[@]}"; do
+        staged_result=$(mktemp /tmp/inferencex-mi325x-result.XXXXXX.json)
+        sudo cp -f "$result_file" "$staged_result"
+        sudo chown "$USER" "$staged_result"
+        mv -f "$staged_result" "$GITHUB_WORKSPACE/$(basename "$result_file")"
+    done
+
+    raw_dir=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -path '*/agentic' -print -quit 2>/dev/null || true)
+    if [[ -n "$raw_dir" ]]; then
+        staged_raw=$(mktemp -d /tmp/inferencex-mi325x-agentic.XXXXXX)
+        sudo cp -a "$raw_dir/." "$staged_raw/"
+        sudo chown -R "$USER" "$staged_raw"
+        mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic"
+        cp -a "$staged_raw/." "$GITHUB_WORKSPACE/LOGS/agentic/"
+        rm -rf -- "$staged_raw"
+    fi
+else
+    while IFS= read -r -d '' result_file; do
+        output="$GITHUB_WORKSPACE/${RESULT_FILENAME}_$(basename "$result_file")"
+        staged_result=$(mktemp /tmp/inferencex-mi325x-result.XXXXXX.json)
+        sudo cp -f "$result_file" "$staged_result"
+        sudo chown "$USER" "$staged_result"
+        mv -f "$staged_result" "$output"
+    done < <(find "$BENCHMARK_LOGS_DIR/logs" -type f -name '*.json' -print0 2>/dev/null)
+fi
+
+stage_server_logs
+scancel_sync "$JOB_ID" || true
+JOB_ID=""
+
+if [[ "${job_state:-}" != COMPLETED* || "${job_exit:-1:0}" != "0:0" ]]; then
+    echo "ERROR: Slurm benchmark failed: state=${job_state:-unknown}, exit=${job_exit:-unknown}" >&2
+    exit 1
+fi

From 02f9748fe5da51710ee4c396c1c977960e49d7ee Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 10:47:36 -0400
Subject: [PATCH 07/42] fix(amd): stage GLM-5.2 in shared multi-node cache
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Serialize missing-model downloads into the MI325X shared NFS Hugging Face cache before submitting disaggregated Slurm jobs.

中文：在提交 MI325X 分离式 Slurm 任务前，将缺失的 GLM-5.2 模型串行下载到共享 NFS Hugging Face 缓存，避免多个拓扑重复下载。
---
 runners/launch_mi325x-amds_multinode.sh | 87 +++++++++++++++++++++----
 1 file changed, 76 insertions(+), 11 deletions(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index cbeebbc9b4..1597e3b37d 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -59,22 +59,87 @@ export GPUS_PER_NODE=8
 export IBDEVICES="${IBDEVICES:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8}"
 export MORI_RDMA_TC="${MORI_RDMA_TC:-104}"
 
-if [[ -z "${MODEL_NAME:-}" ]]; then
-    hf_dir="models--${MODEL//\//--}"
-    snapshot_root="$MODEL_PATH/$hf_dir/snapshots"
+resolve_model_name() {
+    local hf_dir="models--${MODEL//\//--}"
+    local snapshot_root="$MODEL_PATH/$hf_dir/snapshots"
+    local snapshot=""
+
     if [[ -d "$snapshot_root" ]]; then
         snapshot=$(find "$snapshot_root" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort | tail -1)
-    else
-        snapshot=""
     fi
     if [[ -n "$snapshot" ]]; then
-        export MODEL_NAME="$hf_dir/snapshots/$snapshot"
-    elif [[ -d "$MODEL_PATH/${MODEL##*/}" ]]; then
-        export MODEL_NAME="${MODEL##*/}"
-    else
-        echo "ERROR: model '$MODEL' is not staged under $MODEL_PATH" >&2
-        exit 1
+        printf '%s\n' "$hf_dir/snapshots/$snapshot"
+        return 0
+    fi
+    if [[ -d "$MODEL_PATH/${MODEL##*/}" ]]; then
+        printf '%s\n' "${MODEL##*/}"
+        return 0
+    fi
+    return 1
+}
+
+stage_model_to_shared_cache() {
+    local hf_dir="models--${MODEL//\//--}"
+    local lock_file="$MODEL_PATH/.${hf_dir}.download.lock"
+    local python_path=""
+    local tool_dir=""
+
+    mkdir -p "$MODEL_PATH"
+    exec 9>"$lock_file"
+    if ! flock -w 18000 9; then
+        echo "ERROR: timed out waiting to stage '$MODEL' under $MODEL_PATH" >&2
+        return 1
+    fi
+
+    # Another topology may have completed the download while this launcher
+    # waited for the shared-cache lock.
+    if resolve_model_name >/dev/null; then
+        flock -u 9
+        exec 9>&-
+        return 0
+    fi
+
+    echo "Staging '$MODEL' into shared cache $MODEL_PATH"
+    if ! python3 -c 'import huggingface_hub' >/dev/null 2>&1; then
+        tool_dir=$(mktemp -d /tmp/inferencex-hf-client.XXXXXX)
+        python3 -m pip install --quiet --disable-pip-version-check \
+            --target "$tool_dir" 'huggingface_hub>=0.30'
+        python_path="$tool_dir"
+    fi
+
+    PYTHONPATH="${python_path}${PYTHONPATH:+:$PYTHONPATH}" \
+        python3 - "$MODEL" "$MODEL_PATH" <<'PY'
+import os
+import sys
+
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id=sys.argv[1],
+    cache_dir=sys.argv[2],
+    token=os.environ.get("HF_TOKEN"),
+)
+PY
+
+    if [[ -n "$tool_dir" ]]; then
+        rm -rf -- "$tool_dir"
+    fi
+    if ! resolve_model_name >/dev/null; then
+        echo "ERROR: download completed but '$MODEL' is unresolved under $MODEL_PATH" >&2
+        flock -u 9
+        exec 9>&-
+        return 1
+    fi
+    flock -u 9
+    exec 9>&-
+}
+
+if [[ -z "${MODEL_NAME:-}" ]]; then
+    if ! MODEL_NAME=$(resolve_model_name); then
+        stage_model_to_shared_cache
+        MODEL_NAME=$(resolve_model_name)
     fi
+    export MODEL_NAME
 fi
 
 runner_tag=${RUNNER_NAME//[^a-zA-Z0-9_.-]/_}

From abf806bbc8a8eb58d47fc9eccb1ec16b6b07f097 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 10:49:18 -0400
Subject: [PATCH 08/42] fix(amd): create shared model cache as runner user
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Use privileged directory creation only when the MI325X NFS cache is absent, then assign ownership to the runner before downloading model files.

中文：仅在 MI325X NFS 缓存目录缺失时使用特权创建目录，并在下载模型文件前立即将目录所有权交给 runner 用户。
---
 runners/launch_mi325x-amds_multinode.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index 1597e3b37d..84c322d6d7 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -84,7 +84,13 @@ stage_model_to_shared_cache() {
     local python_path=""
     local tool_dir=""
 
-    mkdir -p "$MODEL_PATH"
+    if ! mkdir -p "$MODEL_PATH" 2>/dev/null; then
+        sudo install -d -m 0775 -o "$USER" -g "$(id -gn)" "$MODEL_PATH"
+    fi
+    if [[ ! -w "$MODEL_PATH" ]]; then
+        echo "ERROR: shared model cache '$MODEL_PATH' is not writable by $USER" >&2
+        return 1
+    fi
     exec 9>"$lock_file"
     if ! flock -w 18000 9; then
         echo "ERROR: timed out waiting to stage '$MODEL' under $MODEL_PATH" >&2

From 21c83ac6b64dfd84f11c5f89fc554c3773bc6112 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 10:50:35 -0400
Subject: [PATCH 09/42] fix(amd): use Slurm-visible runner cache for MI325X
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Place the shared Hugging Face cache under the common runner-home filesystem already used for Slurm logs, avoiding the root-squashed data export.

中文：将共享 Hugging Face 缓存放到 Slurm 日志已验证可见的 runner 公共主目录文件系统中，绕过启用 root-squash 的数据挂载。
---
 runners/launch_mi325x-amds_multinode.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index 84c322d6d7..f3abee962c 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -52,7 +52,15 @@ trap cleanup EXIT
 export SLURM_ACCOUNT="$USER"
 export SLURM_PARTITION="compute"
 export SLURM_JOB_NAME="benchmark-sglang-disagg.job"
-export HF_HUB_CACHE_MOUNT="/nfsdata/sa/gharunner/gharunners/hf-hub-cache"
+runner_shared_root="$GITHUB_WORKSPACE"
+for _ in 1 2 3 4 5; do
+    runner_shared_root=$(dirname "$runner_shared_root")
+done
+if [[ "$(basename "$runner_shared_root")" != "gharunners" ]]; then
+    echo "ERROR: cannot derive shared runner root from $GITHUB_WORKSPACE" >&2
+    exit 1
+fi
+export HF_HUB_CACHE_MOUNT="${MI325X_SHARED_HF_CACHE:-$runner_shared_root/.inferencex-hf-cache}"
 export MODEL_PATH="$HF_HUB_CACHE_MOUNT"
 export MODEL_YAML_KEY="${MODEL_YAML_KEY:-${MODEL##*/}}"
 export GPUS_PER_NODE=8
@@ -84,9 +92,7 @@ stage_model_to_shared_cache() {
     local python_path=""
     local tool_dir=""
 
-    if ! mkdir -p "$MODEL_PATH" 2>/dev/null; then
-        sudo install -d -m 0775 -o "$USER" -g "$(id -gn)" "$MODEL_PATH"
-    fi
+    mkdir -p "$MODEL_PATH"
     if [[ ! -w "$MODEL_PATH" ]]; then
         echo "ERROR: shared model cache '$MODEL_PATH' is not writable by $USER" >&2
         return 1

From f57746a357177f5950921f2d76852442a3808b16 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:02:32 -0400
Subject: [PATCH 10/42] fix(amd): surface distributed server failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Propagate the multi-node container srun exit code and relay the preserved Slurm stderr through the GitHub job log.

中文：向上传递多节点容器 srun 的退出码，并将保留的 Slurm stderr 输出到 GitHub 任务日志，便于准确定位分布式服务启动失败。
---
 benchmarks/multi_node/amd_utils/job.slurm | 8 +++++++-
 runners/launch_mi325x-amds_multinode.sh   | 7 +++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 85c014fa1a..f7f1e5f251 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -447,6 +447,7 @@ fi
 # Engine-specific container filter for pre-clean
 CONT_FILTER="name=^container_${ENGINE}_"
 
+SERVER_RUN_RC=0
 srun \
   --nodelist="$SELECTED_NODELIST_SRUN" \
   --kill-on-bad-exit=1 \
@@ -622,7 +623,7 @@ DOCKER_EXIT_CODE=\$?
 echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\"
 \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true
 exit \$DOCKER_EXIT_CODE
-"
+" || SERVER_RUN_RC=$?
 
 if [[ "${KEEP_CONTAINERS}" != "1" ]]; then
     srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true'
@@ -634,3 +635,8 @@ if [[ "${KEEP_CONTAINERS}" != "1" ]]; then
         '
     fi
 fi
+
+if [[ "$SERVER_RUN_RC" -ne 0 ]]; then
+    echo "ERROR: distributed server step failed with exit code $SERVER_RUN_RC" >&2
+    exit "$SERVER_RUN_RC"
+fi
diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index f3abee962c..a269825bff 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -194,6 +194,13 @@ poll_pid=$!
 tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null || true
 wait "$poll_pid"
 
+err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.err"
+if [[ -s "$err_file" ]]; then
+    echo "===== Slurm stderr: job $JOB_ID =====" >&2
+    sudo cat "$err_file" >&2 || true
+    echo "===== End Slurm stderr: job $JOB_ID =====" >&2
+fi
+
 job_state=$(sacct -n -X -j "$JOB_ID" --format=State --parsable2 2>/dev/null | head -1 | cut -d'|' -f1)
 job_exit=$(sacct -n -X -j "$JOB_ID" --format=ExitCode --parsable2 2>/dev/null | head -1 | cut -d'|' -f1)
 echo "Slurm job $JOB_ID finished: state=${job_state:-unknown}, exit=${job_exit:-unknown}"

From 23eb89bfee96e031302fac75a33e3a8f5ea92628 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:04:48 -0400
Subject: [PATCH 11/42] fix(amd): submit MI325X jobs from shared workspace
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Stage a per-run repository copy and Slurm logs under the runner-common filesystem proven visible on all compute nodes, then copy artifacts back and clean it.

中文：在所有计算节点均可见的 runner 公共文件系统中暂存每次运行的仓库副本与 Slurm 日志，完成后回传产物并清理，避免节点无法访问 runner 专属工作目录。
---
 runners/launch_mi325x-amds_multinode.sh | 32 ++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index a269825bff..f2bfa11a6a 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+ORIGINAL_GITHUB_WORKSPACE="$GITHUB_WORKSPACE"
+
 scancel_sync() {
     local job_id="$1"
     local deadline=$((SECONDS + 600))
@@ -26,7 +28,7 @@ stage_server_logs() {
         -C "$BENCHMARK_LOGS_DIR" . 2>/dev/null || true
     sudo chown "$USER" "$archive" 2>/dev/null || true
     if [[ -s "$archive" ]]; then
-        mv -f "$archive" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"
+        mv -f "$archive" "$ORIGINAL_GITHUB_WORKSPACE/multinode_server_logs.tar.gz"
     else
         rm -f "$archive"
     fi
@@ -45,6 +47,12 @@ cleanup() {
     if [[ -n "${BENCHMARK_LOGS_PARENT:-}" && -d "$BENCHMARK_LOGS_PARENT" ]]; then
         rmdir "$BENCHMARK_LOGS_PARENT" 2>/dev/null || true
     fi
+    if [[ -n "${SHARED_WORKSPACE:-}" && -d "$SHARED_WORKSPACE" ]]; then
+        sudo rm -rf -- "$SHARED_WORKSPACE" 2>/dev/null || true
+    fi
+    if [[ -n "${SHARED_WORKSPACES_PARENT:-}" && -d "$SHARED_WORKSPACES_PARENT" ]]; then
+        rmdir "$SHARED_WORKSPACES_PARENT" 2>/dev/null || true
+    fi
     exit "$rc"
 }
 trap cleanup EXIT
@@ -155,7 +163,19 @@ if [[ -z "${MODEL_NAME:-}" ]]; then
 fi
 
 runner_tag=${RUNNER_NAME//[^a-zA-Z0-9_.-]/_}
-BENCHMARK_LOGS_PARENT="$(dirname "$GITHUB_WORKSPACE")/.inferencex-benchmark-logs"
+SHARED_WORKSPACES_PARENT="$runner_shared_root/.inferencex-workspaces"
+mkdir -p "$SHARED_WORKSPACES_PARENT"
+SHARED_WORKSPACE=$(mktemp -d "$SHARED_WORKSPACES_PARENT/mi325x-${runner_tag}.XXXXXX")
+rsync -a \
+    --exclude='.git' \
+    --exclude='LOGS' \
+    --exclude='results' \
+    "$ORIGINAL_GITHUB_WORKSPACE/" "$SHARED_WORKSPACE/"
+export SHARED_WORKSPACES_PARENT
+export SHARED_WORKSPACE
+export GITHUB_WORKSPACE="$SHARED_WORKSPACE"
+
+BENCHMARK_LOGS_PARENT="$runner_shared_root/.inferencex-benchmark-logs"
 mkdir -p "$BENCHMARK_LOGS_PARENT"
 BENCHMARK_LOGS_DIR=$(mktemp -d "$BENCHMARK_LOGS_PARENT/mi325x-${runner_tag}.XXXXXX")
 export BENCHMARK_LOGS_PARENT
@@ -214,7 +234,7 @@ if [[ "${IS_AGENTIC:-0}" == "1" ]]; then
         staged_result=$(mktemp /tmp/inferencex-mi325x-result.XXXXXX.json)
         sudo cp -f "$result_file" "$staged_result"
         sudo chown "$USER" "$staged_result"
-        mv -f "$staged_result" "$GITHUB_WORKSPACE/$(basename "$result_file")"
+        mv -f "$staged_result" "$ORIGINAL_GITHUB_WORKSPACE/$(basename "$result_file")"
     done
 
     raw_dir=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -path '*/agentic' -print -quit 2>/dev/null || true)
@@ -222,13 +242,13 @@ if [[ "${IS_AGENTIC:-0}" == "1" ]]; then
         staged_raw=$(mktemp -d /tmp/inferencex-mi325x-agentic.XXXXXX)
         sudo cp -a "$raw_dir/." "$staged_raw/"
         sudo chown -R "$USER" "$staged_raw"
-        mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic"
-        cp -a "$staged_raw/." "$GITHUB_WORKSPACE/LOGS/agentic/"
+        mkdir -p "$ORIGINAL_GITHUB_WORKSPACE/LOGS/agentic"
+        cp -a "$staged_raw/." "$ORIGINAL_GITHUB_WORKSPACE/LOGS/agentic/"
         rm -rf -- "$staged_raw"
     fi
 else
     while IFS= read -r -d '' result_file; do
-        output="$GITHUB_WORKSPACE/${RESULT_FILENAME}_$(basename "$result_file")"
+        output="$ORIGINAL_GITHUB_WORKSPACE/${RESULT_FILENAME}_$(basename "$result_file")"
         staged_result=$(mktemp /tmp/inferencex-mi325x-result.XXXXXX.json)
         sudo cp -f "$result_file" "$staged_result"
         sudo chown "$USER" "$staged_result"

From a725a9876034d0a0ef8db9684433c4406c14894f Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:12:36 -0400
Subject: [PATCH 12/42] fix(amd): wait for Slurm logs across NFS
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Allow NFS metadata to settle after a fast batch exit, then relay stderr and extended accounting before cleaning the per-run directory.

中文：Slurm 批任务快速退出后等待 NFS 元数据同步，再输出 stderr 与扩展计费状态，最后清理本次运行目录，避免丢失真实错误。
---
 runners/launch_mi325x-amds_multinode.sh | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index f2bfa11a6a..7647d42db3 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -196,10 +196,28 @@ if ! [[ "$JOB_ID" =~ ^[0-9]+$ ]]; then
 fi
 
 log_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out"
+err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.err"
 while [[ ! -f "$log_file" ]]; do
     if ! squeue -h -j "$JOB_ID" 2>/dev/null | grep -q .; then
+        echo "Slurm job $JOB_ID ended before its output log became visible; waiting for NFS metadata" >&2
+        for _ in 1 2 3 4 5 6; do
+            sync
+            [[ -f "$log_file" || -f "$err_file" ]] && break
+            sleep 5
+        done
+        if [[ -f "$log_file" ]]; then
+            break
+        fi
+        if [[ -s "$err_file" ]]; then
+            echo "===== Slurm stderr: job $JOB_ID =====" >&2
+            sudo cat "$err_file" >&2 || true
+            echo "===== End Slurm stderr: job $JOB_ID =====" >&2
+        fi
         echo "ERROR: Slurm job $JOB_ID ended before creating its output log" >&2
         scontrol show job "$JOB_ID" 2>/dev/null || true
+        sacct -X -j "$JOB_ID" \
+            --format=JobID,State,ExitCode,DerivedExitCode,NodeList,Comment,SystemComment \
+            2>/dev/null || true
         exit 1
     fi
     sleep 5
@@ -214,7 +232,6 @@ poll_pid=$!
 tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null || true
 wait "$poll_pid"
 
-err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.err"
 if [[ -s "$err_file" ]]; then
     echo "===== Slurm stderr: job $JOB_ID =====" >&2
     sudo cat "$err_file" >&2 || true

From 42735ff8befa8aa05905c2903d584ba4fca657b9 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:17:43 -0400
Subject: [PATCH 13/42] fix(amd): avoid stale login profiles in Slurm tasks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Run distributed container tasks with a non-login host shell and report the resolved Docker image before launch.\n\n中文：在 Slurm 分布式容器任务中使用非登录宿主机 shell，避免加载陈旧的 runner profile；启动前输出最终解析的 Docker 镜像。
---
 benchmarks/multi_node/amd_utils/job.slurm | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index f7f1e5f251..d19a767aaa 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -453,7 +453,7 @@ srun \
   --kill-on-bad-exit=1 \
   --signal=TERM@30 \
   --unbuffered \
-  bash -lc "
+  bash -c "
 set -euo pipefail
 
 echo \"Rank \$SLURM_PROCID on \$(hostname)\"
@@ -544,6 +544,8 @@ fi  # end: if ENGINE == atom-disagg
 \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true
 \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true
 
+echo \"[docker-run] image=$DOCKER_IMAGE_NAME container=$DOCKER_CONT_NAME\"
+
 # Start vLLM external router container on node 0
 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then
     \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true

From a8540f2edcc083bc774e4880bbc97b1012a0d180 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:21:18 -0400
Subject: [PATCH 14/42] chore(amd): trace distributed Docker launch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Emit the resolved Docker argv for MI325X distributed launch failures so malformed arguments can be identified from Slurm stderr.\n\n中文：在 MI325X 分布式启动失败时输出最终 Docker 参数，便于从 Slurm stderr 中定位格式错误的参数。
---
 benchmarks/multi_node/amd_utils/job.slurm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index d19a767aaa..a3ea29699c 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -576,6 +576,7 @@ if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \
     set +e
 fi
 
+set -x
 \$MAYBE_EXEC \$DOCKER_CMD run \
     --init \
     --stop-timeout 10 \

From 29a3a199b1ce0a00ea19e4de3e50aee44c096b8d Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 11:24:51 -0400
Subject: [PATCH 15/42] fix(amd): preserve metadata in Docker environment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pass space-bearing metadata through Docker environment inheritance so JSON remains a single value in distributed Slurm launches. Remove the temporary argv trace.\n\n中文：通过 Docker 环境变量继承传递包含空格的元数据，确保 JSON 在 Slurm 分布式启动中保持为单一参数；同时移除临时参数跟踪。
---
 benchmarks/multi_node/amd_utils/job.slurm | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index a3ea29699c..a1ce7234ee 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -374,7 +374,7 @@ DOCKER_ENV_COMMON=(
     -e WS_PATH=${WS_PATH}
     -e RUN_EVAL=\$RUN_EVAL
     -e EVAL_ONLY=\$EVAL_ONLY
-    -e \"EVAL_CONC=\$EVAL_CONC\"
+    -e EVAL_CONC
     -e FRAMEWORK=\$FRAMEWORK
     -e PRECISION=\$PRECISION
     -e MODEL_PREFIX=\$MODEL_PREFIX
@@ -392,12 +392,12 @@ DOCKER_ENV_COMMON=(
     -e SCENARIO_TYPE=\$SCENARIO_TYPE
     -e IS_AGENTIC=\$IS_AGENTIC
     -e CONC=\$CONC
-    -e "CONC_LIST=\$CONC_LIST"
+    -e CONC_LIST
     -e DURATION=\$DURATION
     -e KV_OFFLOADING=\$KV_OFFLOADING
     -e KV_OFFLOAD_BACKEND=\$KV_OFFLOAD_BACKEND
-    -e "KV_OFFLOAD_BACKEND_METADATA=\$KV_OFFLOAD_BACKEND_METADATA"
-    -e "ROUTER_METADATA=\$ROUTER_METADATA"
+    -e KV_OFFLOAD_BACKEND_METADATA
+    -e ROUTER_METADATA
     -e KV_P2P_TRANSFER=\$KV_P2P_TRANSFER
     -e TOTAL_CPU_DRAM_GB=\$TOTAL_CPU_DRAM_GB
     -e MAX_MODEL_LEN=\$MAX_MODEL_LEN
@@ -576,7 +576,6 @@ if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \
     set +e
 fi
 
-set -x
 \$MAYBE_EXEC \$DOCKER_CMD run \
     --init \
     --stop-timeout 10 \

From a4ceccd392e667a260282e8ce38c377e0581e8eb Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:16:23 -0400
Subject: [PATCH 16/42] fix(amd): prepare MI325X disaggregated nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pass runner-selected RDMA settings into distributed SGLang containers, pre-pull the image on every allocated node before the creation barrier, and persist server logs in the shared benchmark directory.\n\n中文：将 runner 选择的 RDMA 配置传入 SGLang 分布式容器，在容器创建屏障前为所有分配节点预拉取镜像，并将服务端日志持久化到共享基准测试目录。
---
 benchmarks/multi_node/amd_utils/job.slurm | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index a1ce7234ee..ca6ba6abab 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -404,6 +404,8 @@ DOCKER_ENV_COMMON=(
     -e MODEL=\$MODEL
     -e WEKA_LOADER_OVERRIDE=\$WEKA_LOADER_OVERRIDE
     -e HF_HUB_CACHE=/models
+    -e IBDEVICES
+    -e MORI_RDMA_TC
 )
 
 # Engine-specific env vars
@@ -447,6 +449,20 @@ fi
 # Engine-specific container filter for pre-clean
 CONT_FILTER="name=^container_${ENGINE}_"
 
+echo "Ensuring Docker image is present on every selected node..."
+srun \
+  --nodelist="$SELECTED_NODELIST_SRUN" \
+  --kill-on-bad-exit=1 \
+  --unbuffered \
+  bash -c '
+set -euo pipefail
+eval "$DOCKER_CMD_DETECT"
+if ! $DOCKER_CMD image inspect "$DOCKER_IMAGE_NAME" >/dev/null 2>&1; then
+    echo "[docker-pull] $(hostname): $DOCKER_IMAGE_NAME"
+    $DOCKER_CMD pull "$DOCKER_IMAGE_NAME"
+fi
+'
+
 SERVER_RUN_RC=0
 srun \
   --nodelist="$SELECTED_NODELIST_SRUN" \
@@ -605,7 +621,7 @@ fi
     -v ${MODEL_DIR}:/models \
     -v \$HOME/.ssh:/root/.ssh \
     --shm-size 128G \
-    -v /tmp:/run_logs \
+    -v ${BENCHMARK_LOGS_DIR}:/run_logs \
     -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \
     -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \
     ${EXTRA_DOCKER_MOUNTS:-} \

From 205ebdec0cb09e1a5c94b2a19efab73f7d53ddcb Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:23:16 -0400
Subject: [PATCH 17/42] fix(amd): print failed SGLang server logs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When a distributed SGLang process dies during a blocking startup wait, print its local prefill or decode log before tearing down the Slurm step.\n\n中文：分布式 SGLang 进程在启动等待阶段退出时，先输出对应节点的预填充或解码日志，再终止 Slurm 任务步骤。
---
 benchmarks/multi_node/amd_utils/server_sglang.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index 529eb8b900..d92f65188d 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -420,6 +420,14 @@ wait_or_die() {            # $1 = server pid to watch; rest = blocking command
     while kill -0 "$cmd" 2>/dev/null; do
         kill -0 "$watch" 2>/dev/null || {
             echo "FATAL: $(hostname) local sglang server (pid $watch) died; tearing down job" >&2
+            local log_dir="/run_logs/slurm_job-${SLURM_JOB_ID}"
+            local log_file
+            for log_file in "$log_dir"/*_"$(hostname)".log; do
+                [[ -f "$log_file" ]] || continue
+                echo "===== Local SGLang log: $log_file =====" >&2
+                tail -n 200 "$log_file" >&2 || true
+                echo "===== End local SGLang log =====" >&2
+            done
             kill "$cmd" 2>/dev/null || true
             return 1
         }

From 591986dfa0fdb24c0b3cb2f2838d9fc5cfd86b07 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:29:40 -0400
Subject: [PATCH 18/42] fix(amd): use supported NSA backend flags for GLM-5.2
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the unsupported DSA backend options with the NSA prefill/decode option names exposed by the MI325X disaggregated SGLang image.\n\n中文：将不受支持的 DSA 后端参数替换为 MI325X 分离式 SGLang 镜像实际提供的 NSA 预填充/解码参数名。
---
 benchmarks/multi_node/amd_utils/models.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml
index 47c36f4d9b..03a22374bb 100644
--- a/benchmarks/multi_node/amd_utils/models.yaml
+++ b/benchmarks/multi_node/amd_utils/models.yaml
@@ -265,7 +265,7 @@ GLM-5-FP8:
       cuda_graph_bs_range: "1-128"
 
 GLM-5.2-FP8:
-  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --dsa-topk-backend torch --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
+  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --nsa-prefill-backend tilelang --nsa-decode-backend tilelang --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
   mtp_flags: ""
   dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
   ep_flags: "--moe-a2a-backend mori"

From db33afe28db4442b5be3bfb170f11382072ebcee Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:38:01 -0400
Subject: [PATCH 19/42] fix(amd): report distributed SGLang exit status
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reap failed SGLang server processes, report their return code, and allow process-substitution logs to flush before printing the local failure log.\n\n中文：回收失败的 SGLang 服务进程并输出返回码，同时等待进程替换日志完成刷新后再打印本地故障日志。
---
 benchmarks/multi_node/amd_utils/server_sglang.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index d92f65188d..7937724932 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -419,7 +419,10 @@ wait_or_die() {            # $1 = server pid to watch; rest = blocking command
     "$@" & local cmd=$!
     while kill -0 "$cmd" 2>/dev/null; do
         kill -0 "$watch" 2>/dev/null || {
-            echo "FATAL: $(hostname) local sglang server (pid $watch) died; tearing down job" >&2
+            local server_rc=0
+            wait "$watch" || server_rc=$?
+            sleep 1
+            echo "FATAL: $(hostname) local sglang server (pid $watch) died with rc=$server_rc; tearing down job" >&2
             local log_dir="/run_logs/slurm_job-${SLURM_JOB_ID}"
             local log_file
             for log_file in "$log_dir"/*_"$(hostname)".log; do

From e7299dc29ac88be4b03ce123dc102a87a89350a2 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:45:29 -0400
Subject: [PATCH 20/42] fix(amd): apply GLM-5.2 DSA fallbacks to disaggregation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mirror the working MI325X single-node top-k fallbacks in the disaggregated launcher and enable informative startup logging for GLM-5.2 worker initialization.\n\n中文：在分离式启动器中复用 MI325X 单节点已验证的 top-k 回退配置，并为 GLM-5.2 worker 初始化启用更详细的启动日志。
---
 benchmarks/multi_node/amd_utils/models.yaml      | 2 +-
 benchmarks/multi_node/amd_utils/server_sglang.sh | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml
index 03a22374bb..cee76cfe61 100644
--- a/benchmarks/multi_node/amd_utils/models.yaml
+++ b/benchmarks/multi_node/amd_utils/models.yaml
@@ -265,7 +265,7 @@ GLM-5-FP8:
       cuda_graph_bs_range: "1-128"
 
 GLM-5.2-FP8:
-  base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --nsa-prefill-backend tilelang --nsa-decode-backend tilelang --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
+  base_flags: "--decode-log-interval 1000 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --nsa-prefill-backend tilelang --nsa-decode-backend tilelang --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
   mtp_flags: ""
   dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
   ep_flags: "--moe-a2a-backend mori"
diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index 7937724932..9a78e44972 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -176,6 +176,14 @@ print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"')
 
 echo "Loaded model configuration for: $MODEL_CONFIG_KEY"
 
+if [[ "$MODEL_CONFIG_KEY" == "GLM-5.2-FP8" ]]; then
+    # Match the working MI325X single-node recipe: the fused DSA top-k JIT
+    # includes CUDA-only headers when compiled for gfx942.
+    export SGLANG_DSA_FUSE_TOPK=false
+    export SGLANG_OPT_USE_TOPK_V2=false
+    export PYTHONUNBUFFERED=1
+fi
+
 # Compute DP-dependent prefill parameters
 if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then
     prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP)

From 39a5b1c6fd85a7b04241730bcf01e93afe587044 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 14:54:38 -0400
Subject: [PATCH 21/42] fix(amd): use current SGLang for GLM-5.2 disaggregation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Use the same ROCm 7.2 SGLang 0.5.15 image as the successful single-node run. This published image includes MoRI and Broadcom userspace support while adding current GLM-5.2 runtime support.

中文：GLM-5.2 分离式推理改用当前版本的 SGLang

使用已通过单节点测试的 ROCm 7.2 SGLang 0.5.15 镜像。该公开镜像同时包含 MoRI、Broadcom 用户态支持以及当前 GLM-5.2 运行时支持。
---
 configs/amd-master.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index ace8b76ff6..20a497994a 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2077,7 +2077,7 @@ glm5.2-fp8-mi325x-sglang-agentic:
 # Broadcom bnxt_re MoRI all-to-all path is still unstable, so this sweep varies
 # TP8 prefill/decode worker ratios without EP or DP attention.
 glm5.2-fp8-mi325x-sglang-disagg-agentic:
-  image: semianalysiswork/sgl-bnxt-cdna3:pr22665-bnxt
+  image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x
   model: zai-org/GLM-5.2-FP8
   model-prefix: glm5.2
   runner: cluster:mi325x-amds

From 14968af20ac6f489b0bd235d6c269ccdf4e17ca2 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:08:22 -0400
Subject: [PATCH 22/42] fix(amd): use portable DSA top-k for GLM-5.2
 disaggregation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Match the successful MI325X single-node recipe by selecting TileLang DSA attention and the Torch top-k backend. This avoids the sgl-kernel CUDA-only top-k path on gfx942.

中文：GLM-5.2 分离式推理改用可移植的 DSA top-k 后端

与已通过测试的 MI325X 单节点方案保持一致，DSA attention 使用 TileLang，top-k 使用 Torch 后端，避免 gfx942 上 sgl-kernel 的 CUDA 专用 top-k 路径。
---
 benchmarks/multi_node/amd_utils/models.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml
index cee76cfe61..d23d38b771 100644
--- a/benchmarks/multi_node/amd_utils/models.yaml
+++ b/benchmarks/multi_node/amd_utils/models.yaml
@@ -265,7 +265,7 @@ GLM-5-FP8:
       cuda_graph_bs_range: "1-128"
 
 GLM-5.2-FP8:
-  base_flags: "--decode-log-interval 1000 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --nsa-prefill-backend tilelang --nsa-decode-backend tilelang --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
+  base_flags: "--decode-log-interval 1000 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --served-model-name zai-org/GLM-5.2-FP8 --enable-metrics --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --dsa-topk-backend torch --kv-cache-dtype bfloat16 --context-length 1048576 --max-total-tokens 1048576 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'"
   mtp_flags: ""
   dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head"
   ep_flags: "--moe-a2a-backend mori"

From ea151c0b33e6c5fd219e2c4b04f841196f6bfc58 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:16:51 -0400
Subject: [PATCH 23/42] chore(amd): surface SGLang scheduler initialization
 errors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mirror GLM-5.2 scheduler exceptions to synchronous stderr before SGLang tears down its process tree, so MI325X Slurm logs preserve the actual initialization failure.

中文：输出 SGLang scheduler 初始化错误

在 SGLang 清理进程树之前，将 GLM-5.2 scheduler 异常同步写入 stderr，使 MI325X Slurm 日志能够保留真实的初始化失败原因。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index 35eaf17dc0..3a68fe8899 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -185,6 +185,53 @@ install_transformers_glm5() {
     _SETUP_INSTALLED+=("transformers-glm5")
 }
 
+# ---------------------------------------------------------------------------
+# SGLang: write scheduler initialization exceptions synchronously.
+#
+# The scheduler normally emits through Python logging and immediately signals
+# the parent to tear down the process tree. On the MI325X Slurm path that can
+# terminate the container before the buffered traceback reaches the job log.
+# Keep the normal error handling, but mirror the traceback to stderr first.
+# ---------------------------------------------------------------------------
+patch_sglang_scheduler_traceback() {
+    if [[ "${MODEL_CONFIG_KEY:-}" != "GLM-5.2-FP8" ]]; then
+        return 0
+    fi
+
+    python3 -c '
+import os, sys
+
+target = "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py"
+if not os.path.isfile(target):
+    print("[SETUP] SGLang scheduler.py not found, skipping traceback patch")
+    sys.exit(0)
+
+src = open(target).read()
+marker = "SGLANG_SCHEDULER_INIT_TRACEBACK"
+if marker in src:
+    print("[SETUP] synchronous scheduler traceback patch already applied")
+    sys.exit(0)
+
+old = """\
+        traceback = get_exception_traceback()
+        logger.error(f"Scheduler hit an exception: {traceback}")
+        parent_process.send_signal(signal.SIGQUIT)"""
+new = """\
+        traceback = get_exception_traceback()
+        logger.error(f"Scheduler hit an exception: {traceback}")
+        os.write(2, ("SGLANG_SCHEDULER_INIT_TRACEBACK\\n" + traceback + "\\n").encode("utf-8", errors="replace"))
+        parent_process.send_signal(signal.SIGQUIT)"""
+
+if old not in src:
+    print("[SETUP] WARN: SGLang scheduler exception pattern not found")
+    sys.exit(0)
+
+open(target, "w").write(src.replace(old, new, 1))
+print("[SETUP] Patched: synchronous scheduler initialization traceback")
+'
+    _SETUP_INSTALLED+=("scheduler-init-traceback")
+}
+
 # =============================================================================
 # Run installers (engine-gated)
 # =============================================================================
@@ -204,6 +251,7 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
 else
     patch_gluon_pa_mqa_logits_instr_shape
     install_transformers_glm5
+    patch_sglang_scheduler_traceback
 fi
 
 _SETUP_END=$(date +%s)

From 06aa78cc1bcd90e272cc1e225c069bc4ce383894 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:24:42 -0400
Subject: [PATCH 24/42] chore(amd): trace scheduler process setup failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wrap the GLM-5.2 scheduler multiprocessing target so failures before SGLang enters its internal exception handler are written synchronously to stderr.

中文：追踪 scheduler 进程启动失败

为 GLM-5.2 scheduler 的 multiprocessing 入口增加外层异常输出，使 SGLang 内部异常处理器启动前的失败也能同步写入 stderr。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index 3a68fe8899..f0a754d546 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -228,6 +228,52 @@ if old not in src:
 
 open(target, "w").write(src.replace(old, new, 1))
 print("[SETUP] Patched: synchronous scheduler initialization traceback")
+'
+
+    python3 -c '
+import os, sys
+
+target = "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py"
+if not os.path.isfile(target):
+    print("[SETUP] SGLang engine.py not found, skipping process traceback patch")
+    sys.exit(0)
+
+src = open(target).read()
+marker = "SGLANG_SCHEDULER_PROCESS_TRACEBACK"
+if marker in src:
+    print("[SETUP] scheduler process traceback patch already applied")
+    sys.exit(0)
+
+class_marker = "class Engine:"
+old_target = """\
+                            target=run_scheduler_process_func,
+                            args=(
+                                server_args,"""
+wrapper = """\
+def _inferencex_run_scheduler_with_traceback(target, *args):
+    try:
+        return target(*args)
+    except BaseException:
+        trace = __import__("traceback").format_exc()
+        os.write(2, ("SGLANG_SCHEDULER_PROCESS_TRACEBACK\\n" + trace + "\\n").encode("utf-8", errors="replace"))
+        raise
+
+
+"""
+new_target = """\
+                            target=_inferencex_run_scheduler_with_traceback,
+                            args=(
+                                run_scheduler_process_func,
+                                server_args,"""
+
+if class_marker not in src or old_target not in src:
+    print("[SETUP] WARN: SGLang engine scheduler target pattern not found")
+    sys.exit(0)
+
+src = src.replace(class_marker, wrapper + class_marker, 1)
+src = src.replace(old_target, new_target, 1)
+open(target, "w").write(src)
+print("[SETUP] Patched: scheduler process target traceback")
 '
     _SETUP_INSTALLED+=("scheduler-init-traceback")
 }

From 90c5e85d44d6c7706e5fd24abd04356a09cc88c5 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:30:54 -0400
Subject: [PATCH 25/42] chore(amd): match SGLang scheduler target layout
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Patch the stable multiprocessing target assignment used by the published ROCm image so pre-handler scheduler failures are captured synchronously.

中文：适配 SGLang scheduler 入口布局

针对公开 ROCm 镜像中稳定的 multiprocessing target 赋值位置应用补丁，从而同步捕获内部异常处理器启动前的 scheduler 失败。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index f0a754d546..b1c9456e6d 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -245,14 +245,12 @@ if marker in src:
     sys.exit(0)
 
 class_marker = "class Engine:"
-old_target = """\
-                            target=run_scheduler_process_func,
-                            args=(
-                                server_args,"""
+old_target = "target=run_scheduler_process_func,"
 wrapper = """\
-def _inferencex_run_scheduler_with_traceback(target, *args):
+def _inferencex_run_scheduler_with_traceback(*args, **kwargs):
+    from sglang.srt.managers.scheduler import run_scheduler_process
     try:
-        return target(*args)
+        return run_scheduler_process(*args, **kwargs)
     except BaseException:
         trace = __import__("traceback").format_exc()
         os.write(2, ("SGLANG_SCHEDULER_PROCESS_TRACEBACK\\n" + trace + "\\n").encode("utf-8", errors="replace"))
@@ -260,11 +258,7 @@ def _inferencex_run_scheduler_with_traceback(target, *args):
 
 
 """
-new_target = """\
-                            target=_inferencex_run_scheduler_with_traceback,
-                            args=(
-                                run_scheduler_process_func,
-                                server_args,"""
+new_target = "target=_inferencex_run_scheduler_with_traceback,"
 
 if class_marker not in src or old_target not in src:
     print("[SETUP] WARN: SGLang engine scheduler target pattern not found")

From 14a6314c92276188411a797157081403fd2c6493 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:37:12 -0400
Subject: [PATCH 26/42] chore(amd): locate scheduler target with flexible
 matching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Use whitespace-tolerant matching for the scheduler multiprocessing target in the published ROCm image so the diagnostic wrapper is applied reliably.

中文：灵活匹配 scheduler 进程入口

对公开 ROCm 镜像中的 scheduler multiprocessing 入口使用可容忍空白差异的匹配方式，确保诊断包装器可靠生效。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index b1c9456e6d..f79629ea5b 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -96,7 +96,7 @@ install_amd_quark() {
 # ---------------------------------------------------------------------------
 patch_gluon_pa_mqa_logits_instr_shape() {
     python3 -c '
-import os, sys
+import os, re, sys
 
 target = "/sgl-workspace/aiter/aiter/ops/triton/gluon/pa_mqa_logits.py"
 if not os.path.isfile(target):
@@ -231,7 +231,7 @@ print("[SETUP] Patched: synchronous scheduler initialization traceback")
 '
 
     python3 -c '
-import os, sys
+import os, re, sys
 
 target = "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py"
 if not os.path.isfile(target):
@@ -244,8 +244,8 @@ if marker in src:
     print("[SETUP] scheduler process traceback patch already applied")
     sys.exit(0)
 
-class_marker = "class Engine:"
-old_target = "target=run_scheduler_process_func,"
+class_match = re.search(r"^class Engine(?:\\(|:)", src, re.MULTILINE)
+target_match = re.search(r"target\\s*=\\s*run_scheduler_process_func\\s*,", src)
 wrapper = """\
 def _inferencex_run_scheduler_with_traceback(*args, **kwargs):
     from sglang.srt.managers.scheduler import run_scheduler_process
@@ -258,14 +258,17 @@ def _inferencex_run_scheduler_with_traceback(*args, **kwargs):
 
 
 """
-new_target = "target=_inferencex_run_scheduler_with_traceback,"
-
-if class_marker not in src or old_target not in src:
+if class_match is None or target_match is None:
     print("[SETUP] WARN: SGLang engine scheduler target pattern not found")
     sys.exit(0)
 
-src = src.replace(class_marker, wrapper + class_marker, 1)
-src = src.replace(old_target, new_target, 1)
+src = src[:class_match.start()] + wrapper + src[class_match.start():]
+src = re.sub(
+    r"target\\s*=\\s*run_scheduler_process_func\\s*,",
+    "target=_inferencex_run_scheduler_with_traceback,",
+    src,
+    count=1,
+)
 open(target, "w").write(src)
 print("[SETUP] Patched: scheduler process target traceback")
 '

From f80e594b060a604092301a99cb187c3a35080a69 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:44:16 -0400
Subject: [PATCH 27/42] chore(amd): fix scheduler trace matching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Correct the Python raw regexes used to instrument the SGLang scheduler process so the diagnostic wrapper is actually installed.\n\n中文：修正用于插桩 SGLang 调度器进程的 Python 原始正则表达式，确保诊断包装器能够正确安装。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index f79629ea5b..ae2855941b 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -244,8 +244,8 @@ if marker in src:
     print("[SETUP] scheduler process traceback patch already applied")
     sys.exit(0)
 
-class_match = re.search(r"^class Engine(?:\\(|:)", src, re.MULTILINE)
-target_match = re.search(r"target\\s*=\\s*run_scheduler_process_func\\s*,", src)
+class_match = re.search(r"^class Engine(?:\(|:)", src, re.MULTILINE)
+target_match = re.search(r"target\s*=\s*run_scheduler_process_func\s*,", src)
 wrapper = """\
 def _inferencex_run_scheduler_with_traceback(*args, **kwargs):
     from sglang.srt.managers.scheduler import run_scheduler_process
@@ -264,7 +264,7 @@ if class_match is None or target_match is None:
 
 src = src[:class_match.start()] + wrapper + src[class_match.start():]
 src = re.sub(
-    r"target\\s*=\\s*run_scheduler_process_func\\s*,",
+    r"target\s*=\s*run_scheduler_process_func\s*,",
     "target=_inferencex_run_scheduler_with_traceback,",
     src,
     count=1,

From 617027a7389de76dda63c95165bd7e8af510db44 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 15:55:18 -0400
Subject: [PATCH 28/42] chore(amd): capture top-level SGLang launch failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mirror uncaught launch exceptions directly to stderr and enable Python fatal-signal traces for the GLM-5.2 disaggregated startup investigation.\n\n中文：将未捕获的启动异常直接同步输出到 stderr，并启用 Python 致命信号追踪，用于排查 GLM-5.2 分离式推理启动失败。
---
 .../multi_node/amd_utils/server_sglang.sh     |  1 +
 benchmarks/multi_node/amd_utils/setup_deps.sh | 37 +++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index 9a78e44972..6a80529da8 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -11,6 +11,7 @@ NODE_RANK="${NODE_RANK:-0}"
 MODEL_DIR="${MODEL_DIR:-}"
 MODEL_NAME="${MODEL_NAME:-}"
 MODEL_CONFIG_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
+export PYTHONFAULTHANDLER=1
 
 xP="${xP:-1}" #-> Number of Prefill Workers
 yD="${yD:-1}" #-> Number of Decode Workers
diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index ae2855941b..88b912941b 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -271,6 +271,43 @@ src = re.sub(
 )
 open(target, "w").write(src)
 print("[SETUP] Patched: scheduler process target traceback")
+'
+
+    python3 -c '
+import os, sys
+
+target = "/sgl-workspace/sglang/python/sglang/launch_server.py"
+if not os.path.isfile(target):
+    print("[SETUP] SGLang launch_server.py not found, skipping top-level traceback patch")
+    sys.exit(0)
+
+src = open(target).read()
+marker = "SGLANG_LAUNCH_SERVER_TRACEBACK"
+if marker in src:
+    print("[SETUP] top-level launch traceback patch already applied")
+    sys.exit(0)
+
+old = """\
+    try:
+        run_server(server_args)
+    finally:
+        kill_process_tree(os.getpid(), include_parent=False)"""
+new = """\
+    try:
+        run_server(server_args)
+    except BaseException:
+        trace = __import__("traceback").format_exc()
+        os.write(2, ("SGLANG_LAUNCH_SERVER_TRACEBACK\\n" + trace + "\\n").encode("utf-8", errors="replace"))
+        raise
+    finally:
+        kill_process_tree(os.getpid(), include_parent=False)"""
+
+if old not in src:
+    print("[SETUP] WARN: SGLang top-level launch pattern not found")
+    sys.exit(0)
+
+open(target, "w").write(src.replace(old, new, 1))
+print("[SETUP] Patched: top-level launch traceback")
 '
     _SETUP_INSTALLED+=("scheduler-init-traceback")
 }

From f1efd552fa59d3819aa4e8c5626a5b818bc74aa9 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 16:03:55 -0400
Subject: [PATCH 29/42] fix(amd): install GLM-5.2 tokenizer dependencies
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Install sentencepiece and tiktoken for GLM-5.2 disaggregated workers, which otherwise fail before model loading while constructing the tokenizer. Remove the temporary traceback instrumentation now that the root cause is identified.\n\n中文：为 GLM-5.2 分离式推理工作进程安装 sentencepiece 和 tiktoken，避免在模型加载前构建分词器时失败。根因已确认，同时移除临时异常追踪插桩。
---
 .../multi_node/amd_utils/server_sglang.sh     |   1 -
 benchmarks/multi_node/amd_utils/setup_deps.sh | 129 ++----------------
 2 files changed, 10 insertions(+), 120 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index 6a80529da8..9a78e44972 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -11,7 +11,6 @@ NODE_RANK="${NODE_RANK:-0}"
 MODEL_DIR="${MODEL_DIR:-}"
 MODEL_NAME="${MODEL_NAME:-}"
 MODEL_CONFIG_KEY="${MODEL_YAML_KEY:-$MODEL_NAME}"
-export PYTHONFAULTHANDLER=1
 
 xP="${xP:-1}" #-> Number of Prefill Workers
 yD="${yD:-1}" #-> Number of Decode Workers
diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index 88b912941b..33f78a0813 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -186,130 +186,21 @@ install_transformers_glm5() {
 }
 
 # ---------------------------------------------------------------------------
-# SGLang: write scheduler initialization exceptions synchronously.
-#
-# The scheduler normally emits through Python logging and immediately signals
-# the parent to tear down the process tree. On the MI325X Slurm path that can
-# terminate the container before the buffered traceback reaches the job log.
-# Keep the normal error handling, but mirror the traceback to stderr first.
+# SGLang: install GLM-5.2 tokenizer conversion dependencies.
 # ---------------------------------------------------------------------------
-patch_sglang_scheduler_traceback() {
+install_glm52_tokenizer_deps() {
     if [[ "${MODEL_CONFIG_KEY:-}" != "GLM-5.2-FP8" ]]; then
         return 0
     fi
 
-    python3 -c '
-import os, sys
-
-target = "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py"
-if not os.path.isfile(target):
-    print("[SETUP] SGLang scheduler.py not found, skipping traceback patch")
-    sys.exit(0)
-
-src = open(target).read()
-marker = "SGLANG_SCHEDULER_INIT_TRACEBACK"
-if marker in src:
-    print("[SETUP] synchronous scheduler traceback patch already applied")
-    sys.exit(0)
-
-old = """\
-        traceback = get_exception_traceback()
-        logger.error(f"Scheduler hit an exception: {traceback}")
-        parent_process.send_signal(signal.SIGQUIT)"""
-new = """\
-        traceback = get_exception_traceback()
-        logger.error(f"Scheduler hit an exception: {traceback}")
-        os.write(2, ("SGLANG_SCHEDULER_INIT_TRACEBACK\\n" + traceback + "\\n").encode("utf-8", errors="replace"))
-        parent_process.send_signal(signal.SIGQUIT)"""
-
-if old not in src:
-    print("[SETUP] WARN: SGLang scheduler exception pattern not found")
-    sys.exit(0)
-
-open(target, "w").write(src.replace(old, new, 1))
-print("[SETUP] Patched: synchronous scheduler initialization traceback")
-'
-
-    python3 -c '
-import os, re, sys
-
-target = "/sgl-workspace/sglang/python/sglang/srt/entrypoints/engine.py"
-if not os.path.isfile(target):
-    print("[SETUP] SGLang engine.py not found, skipping process traceback patch")
-    sys.exit(0)
-
-src = open(target).read()
-marker = "SGLANG_SCHEDULER_PROCESS_TRACEBACK"
-if marker in src:
-    print("[SETUP] scheduler process traceback patch already applied")
-    sys.exit(0)
-
-class_match = re.search(r"^class Engine(?:\(|:)", src, re.MULTILINE)
-target_match = re.search(r"target\s*=\s*run_scheduler_process_func\s*,", src)
-wrapper = """\
-def _inferencex_run_scheduler_with_traceback(*args, **kwargs):
-    from sglang.srt.managers.scheduler import run_scheduler_process
-    try:
-        return run_scheduler_process(*args, **kwargs)
-    except BaseException:
-        trace = __import__("traceback").format_exc()
-        os.write(2, ("SGLANG_SCHEDULER_PROCESS_TRACEBACK\\n" + trace + "\\n").encode("utf-8", errors="replace"))
-        raise
-
-
-"""
-if class_match is None or target_match is None:
-    print("[SETUP] WARN: SGLang engine scheduler target pattern not found")
-    sys.exit(0)
-
-src = src[:class_match.start()] + wrapper + src[class_match.start():]
-src = re.sub(
-    r"target\s*=\s*run_scheduler_process_func\s*,",
-    "target=_inferencex_run_scheduler_with_traceback,",
-    src,
-    count=1,
-)
-open(target, "w").write(src)
-print("[SETUP] Patched: scheduler process target traceback")
-'
-
-    python3 -c '
-import os, sys
-
-target = "/sgl-workspace/sglang/python/sglang/launch_server.py"
-if not os.path.isfile(target):
-    print("[SETUP] SGLang launch_server.py not found, skipping top-level traceback patch")
-    sys.exit(0)
-
-src = open(target).read()
-marker = "SGLANG_LAUNCH_SERVER_TRACEBACK"
-if marker in src:
-    print("[SETUP] top-level launch traceback patch already applied")
-    sys.exit(0)
-
-old = """\
-    try:
-        run_server(server_args)
-    finally:
-        kill_process_tree(os.getpid(), include_parent=False)"""
-new = """\
-    try:
-        run_server(server_args)
-    except BaseException:
-        trace = __import__("traceback").format_exc()
-        os.write(2, ("SGLANG_LAUNCH_SERVER_TRACEBACK\\n" + trace + "\\n").encode("utf-8", errors="replace"))
-        raise
-    finally:
-        kill_process_tree(os.getpid(), include_parent=False)"""
-
-if old not in src:
-    print("[SETUP] WARN: SGLang top-level launch pattern not found")
-    sys.exit(0)
+    if python3 -c 'import sentencepiece, tiktoken' 2>/dev/null; then
+        echo "[SETUP] GLM-5.2 tokenizer dependencies already present"
+        return 0
+    fi
 
-open(target, "w").write(src.replace(old, new, 1))
-print("[SETUP] Patched: top-level launch traceback")
-'
-    _SETUP_INSTALLED+=("scheduler-init-traceback")
+    echo "[SETUP] Installing GLM-5.2 tokenizer dependencies..."
+    pip install --quiet --no-cache-dir sentencepiece tiktoken
+    _SETUP_INSTALLED+=("glm52-tokenizer-deps")
 }
 
 # =============================================================================
@@ -331,7 +222,7 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
 else
     patch_gluon_pa_mqa_logits_instr_shape
     install_transformers_glm5
-    patch_sglang_scheduler_traceback
+    install_glm52_tokenizer_deps
 fi
 
 _SETUP_END=$(date +%s)

From 7295fb1ce31408b3ac93d71ce71dd45147010e2e Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 16:11:34 -0400
Subject: [PATCH 30/42] fix(amd): resume incomplete shared model snapshots
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Require the MI325X multi-node cache snapshot to contain its config, weight index, and tokenizer files before reuse. An incomplete GLM-5.2 snapshot is now resumed through snapshot_download instead of reaching SGLang without tokenizer.json. Remove the unnecessary tokenizer dependency installation.\n\n中文：MI325X 多节点共享缓存快照只有在配置、权重索引和分词器文件齐全时才会复用。若 GLM-5.2 快照不完整，将通过 snapshot_download 续传，避免缺少 tokenizer.json 时直接启动 SGLang；同时移除不必要的分词器依赖安装。
---
 benchmarks/multi_node/amd_utils/setup_deps.sh | 19 -------------------
 runners/launch_mi325x-amds_multinode.sh       | 17 ++++++++++++++++-
 2 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh
index 33f78a0813..b6c46896ad 100644
--- a/benchmarks/multi_node/amd_utils/setup_deps.sh
+++ b/benchmarks/multi_node/amd_utils/setup_deps.sh
@@ -185,24 +185,6 @@ install_transformers_glm5() {
     _SETUP_INSTALLED+=("transformers-glm5")
 }
 
-# ---------------------------------------------------------------------------
-# SGLang: install GLM-5.2 tokenizer conversion dependencies.
-# ---------------------------------------------------------------------------
-install_glm52_tokenizer_deps() {
-    if [[ "${MODEL_CONFIG_KEY:-}" != "GLM-5.2-FP8" ]]; then
-        return 0
-    fi
-
-    if python3 -c 'import sentencepiece, tiktoken' 2>/dev/null; then
-        echo "[SETUP] GLM-5.2 tokenizer dependencies already present"
-        return 0
-    fi
-
-    echo "[SETUP] Installing GLM-5.2 tokenizer dependencies..."
-    pip install --quiet --no-cache-dir sentencepiece tiktoken
-    _SETUP_INSTALLED+=("glm52-tokenizer-deps")
-}
-
 # =============================================================================
 # Run installers (engine-gated)
 # =============================================================================
@@ -222,7 +204,6 @@ if [[ "$ENGINE" == "vllm-disagg" ]]; then
 else
     patch_gluon_pa_mqa_logits_instr_shape
     install_transformers_glm5
-    install_glm52_tokenizer_deps
 fi
 
 _SETUP_END=$(date +%s)
diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index 7647d42db3..592b7f5f12 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -75,6 +75,21 @@ export GPUS_PER_NODE=8
 export IBDEVICES="${IBDEVICES:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8}"
 export MORI_RDMA_TC="${MORI_RDMA_TC:-104}"
 
+snapshot_has_required_files() {
+    local snapshot_dir="$1"
+    local required_file
+    for required_file in \
+        config.json \
+        model.safetensors.index.json \
+        tokenizer.json \
+        tokenizer_config.json; do
+        if [[ ! -f "$snapshot_dir/$required_file" ]]; then
+            echo "Model snapshot is incomplete: missing $snapshot_dir/$required_file" >&2
+            return 1
+        fi
+    done
+}
+
 resolve_model_name() {
     local hf_dir="models--${MODEL//\//--}"
     local snapshot_root="$MODEL_PATH/$hf_dir/snapshots"
@@ -83,7 +98,7 @@ resolve_model_name() {
     if [[ -d "$snapshot_root" ]]; then
         snapshot=$(find "$snapshot_root" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort | tail -1)
     fi
-    if [[ -n "$snapshot" ]]; then
+    if [[ -n "$snapshot" ]] && snapshot_has_required_files "$snapshot_root/$snapshot"; then
         printf '%s\n' "$hf_dir/snapshots/$snapshot"
         return 0
     fi

From 92344f2b08b2504f08b37dc5b8cca21d8de2e171 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 16:16:40 -0400
Subject: [PATCH 31/42] fix(amd): reuse validated GLM-5.2 node cache
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Point MI325X multi-node GLM-5.2 jobs at the exact node-local Hugging Face snapshot already validated by the successful single-node sweep on nodes 017 and 018. This avoids reconstructing the 753 GB checkpoint in the controller-visible cache.\n\n中文：MI325X 多节点 GLM-5.2 任务改为复用节点 017 和 018 上已通过单节点扫描验证的 Hugging Face 精确快照，避免在控制器可见缓存中重新构建 753 GB 检查点。
---
 runners/launch_mi325x-amds_multinode.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/runners/launch_mi325x-amds_multinode.sh b/runners/launch_mi325x-amds_multinode.sh
index 592b7f5f12..53ed489f6c 100755
--- a/runners/launch_mi325x-amds_multinode.sh
+++ b/runners/launch_mi325x-amds_multinode.sh
@@ -69,7 +69,16 @@ if [[ "$(basename "$runner_shared_root")" != "gharunners" ]]; then
     exit 1
 fi
 export HF_HUB_CACHE_MOUNT="${MI325X_SHARED_HF_CACHE:-$runner_shared_root/.inferencex-hf-cache}"
-export MODEL_PATH="$HF_HUB_CACHE_MOUNT"
+if [[ "$MODEL" == "zai-org/GLM-5.2-FP8" ]]; then
+    # Reuse the node-local cache populated and validated by the successful
+    # single-node MI325X AgentX sweep. Both nodes used by the 1P1D allocation
+    # have this exact revision; copying the 753 GB checkpoint into the
+    # controller-visible cache is unnecessary and unreliable through Xet.
+    export MODEL_PATH="${MI325X_NODE_LOCAL_HF_CACHE:-/raid/hf-hub-cache}"
+    export MODEL_NAME="models--zai-org--GLM-5.2-FP8/snapshots/ba978f7d347eaf65d22f1a86833408afdb953541"
+else
+    export MODEL_PATH="$HF_HUB_CACHE_MOUNT"
+fi
 export MODEL_YAML_KEY="${MODEL_YAML_KEY:-${MODEL##*/}}"
 export GPUS_PER_NODE=8
 export IBDEVICES="${IBDEVICES:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re7,bnxt_re8}"

From 70e96b47eeb61f17d2dad316ad7706c19f235c10 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Sun, 19 Jul 2026 19:45:36 -0400
Subject: [PATCH 32/42] fix(amd): allow slower GLM disagg startup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：将 GLM-5.2 MI325X 分离式推理的服务同步等待时间延长到一小时，并将该配置传入各 Slurm 容器，以适配多节点并发加载模型时更长的启动时间。
---
 benchmarks/multi_node/amd_utils/job.slurm        | 1 +
 benchmarks/multi_node/amd_utils/server_sglang.sh | 2 +-
 configs/amd-master.yaml                          | 3 +++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index ca6ba6abab..840062b5fb 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -403,6 +403,7 @@ DOCKER_ENV_COMMON=(
     -e MAX_MODEL_LEN=\$MAX_MODEL_LEN
     -e MODEL=\$MODEL
     -e WEKA_LOADER_OVERRIDE=\$WEKA_LOADER_OVERRIDE
+    -e SGLANG_SERVER_READY_TIMEOUT
     -e HF_HUB_CACHE=/models
     -e IBDEVICES
     -e MORI_RDMA_TC
diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh
index 9a78e44972..ef9370dbb8 100755
--- a/benchmarks/multi_node/amd_utils/server_sglang.sh
+++ b/benchmarks/multi_node/amd_utils/server_sglang.sh
@@ -517,7 +517,7 @@ if [ "$NODE_RANK" -eq 0 ]; then
         --node-ips ${IPADDRS} \
         --node-ports 8000 \
         --wait-for-all-ports \
-        --timeout 1800"
+        --timeout ${SGLANG_SERVER_READY_TIMEOUT:-1800}"
 
     if [[ "$DRY_RUN" -eq 1 ]]; then
         echo "DRY RUN: $BARRIER_CMD"
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 20a497994a..8c46a4d6b7 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2101,6 +2101,7 @@ glm5.2-fp8-mi325x-sglang-disagg-agentic:
           additional-settings:
           - "PREFILL_NODES=1"
           - "MODEL_YAML_KEY=GLM-5.2-FP8"
+          - "SGLANG_SERVER_READY_TIMEOUT=3600"
         decode:
           num-worker: 1
           tp: 8
@@ -2121,6 +2122,7 @@ glm5.2-fp8-mi325x-sglang-disagg-agentic:
           additional-settings:
           - "PREFILL_NODES=1"
           - "MODEL_YAML_KEY=GLM-5.2-FP8"
+          - "SGLANG_SERVER_READY_TIMEOUT=3600"
         decode:
           num-worker: 2
           tp: 8
@@ -2141,6 +2143,7 @@ glm5.2-fp8-mi325x-sglang-disagg-agentic:
           additional-settings:
           - "PREFILL_NODES=2"
           - "MODEL_YAML_KEY=GLM-5.2-FP8"
+          - "SGLANG_SERVER_READY_TIMEOUT=3600"
         decode:
           num-worker: 1
           tp: 8

From 8e89d537d085dc8d7210b38f5dcd5f7b6bf06049 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Mon, 20 Jul 2026 00:48:19 -0400
Subject: [PATCH 33/42] chore: consolidate GLM-5.2 sweep trigger
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：合并本 PR 的 GLM-5.2 MI325X 变更日志条目，确保正式扫描仅执行一次 7 点单节点曲线和一次 12 点多节点曲线，避免重复占用 GPU。
---
 perf-changelog.yaml | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 959f0de9ec..f7d916b72a 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4967,19 +4967,12 @@
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280
 
-- config-keys:
-    - glm5.2-fp8-mi325x-sglang-agentic
-  description:
-    - "Add a GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context qualification"
-    - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
-    - "Use lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x with the AMD DSA TileLang prefill and decode backends; qualify concurrency 1 before expanding the frontier"
-  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273
-
 - config-keys:
     - glm5.2-fp8-mi325x-sglang-agentic
     - glm5.2-fp8-mi325x-sglang-disagg-agentic
   description:
-    - "Expand the single-node GLM-5.2 MI325X full-context AgentX curve to concurrency [1, 2, 3, 4, 5, 6, 8] around the measured KV-capacity knee"
+    - "Add a GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context curve at concurrency [1, 2, 3, 4, 5, 6, 8] around the measured KV-capacity knee"
+    - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
     - "Add TP8 SGLang disaggregated AgentX sweeps for balanced 1P1D, decode-heavy 1P2D, and prefill-heavy 2P1D topologies, with at most four concurrency points per server allocation"
     - "Keep EP and DP attention disabled because MoRI all-to-all remains unstable on the MI325X Broadcom bnxt_re fleet"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273

From f488821fa2e383bed0fe94159be6f9249fe4e1ab Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Mon, 20 Jul 2026 10:24:38 -0400
Subject: [PATCH 34/42] chore: retrigger MI325X AgentX full sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

中文：重新触发 MI325X AgentX 完整扫描。

From 7fd057c2a67e4ed3c5aa844a183d8bfd84ab2091 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Mon, 20 Jul 2026 12:31:27 -0400
Subject: [PATCH 35/42] fix: sanitize AMD multinode model paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sanitize cache-backed model names before using them as Docker container names, look up YAML by the explicit recipe key, and forward that key into the server container.\n\n中文：在将缓存模型路径用于 Docker 容器名称前进行安全化处理，改用显式 recipe key 查询 YAML，并将该 key 传入服务容器。
---
 benchmarks/multi_node/amd_utils/job.slurm | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm
index 760750e50b..fb53557143 100755
--- a/benchmarks/multi_node/amd_utils/job.slurm
+++ b/benchmarks/multi_node/amd_utils/job.slurm
@@ -117,7 +117,7 @@ export MODEL_DIR
 
 if [[ "$ENGINE" == "vllm-disagg" ]]; then
     # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots
-    DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next}
+    DISK_DIR_NAME=$(awk -v key="$MODEL_YAML_KEY" '$0 == key ":" {found=1; next}
         found && /^[^ ]/{exit}
         found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML")
     DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}"
@@ -193,7 +193,7 @@ else
     }
 
     # Extract hf_dir from models.yaml (same as vllm-disagg path above)
-    SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next}
+    SGL_DISK_DIR_NAME=$(awk -v key="$MODEL_YAML_KEY" '$0 == key ":" {found=1; next}
         found && /^[^ ]/{exit}
         found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML")
     SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}"
@@ -348,7 +348,8 @@ export SPEC_DECODING="${SPEC_DECODING:-}"
 export IS_MULTINODE="${IS_MULTINODE:-false}"
 
 SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_')
-export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}"
+SANITIZED_MODEL=$(printf '%s' "$MODEL_NAME" | tr -c 'a-zA-Z0-9_.-' '_')
+export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${SANITIZED_MODEL}_${SLURM_JOB_ID}"
 
 # vLLM external router container.
 # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older
@@ -400,6 +401,7 @@ DOCKER_ENV_COMMON=(
     -e NODE0_ADDR=\$NODE0_ADDR
     -e MODEL_DIR=/models
     -e MODEL_NAME=\$MODEL_NAME
+    -e MODEL_YAML_KEY=\$MODEL_YAML_KEY
     -e GPUS_PER_NODE=\$GPUS_PER_NODE
     -e xP=\$xP
     -e yD=\$yD

From 33edc29e2f68f5ec77ecd2a5cc031c366ef0e362 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Mon, 20 Jul 2026 22:35:22 -0700
Subject: [PATCH 36/42] feat: expand GLM-5.2 MI325X AgentX curves
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add low-latency, EP8 balanced, EP8/DP8 high-throughput, and HiCache single-node profiles while preserving the 1M request context and GPU KV pool.\n\n中文：新增低延迟、EP8 均衡、EP8/DP8 高吞吐和 HiCache 单节点配置，同时保持 1M 请求上下文与 GPU KV 池。
---
 .../single_node/agentic/glm5.2_fp8_mi325x.sh  | 105 +++++++++++++++---
 configs/amd-master.yaml                       |  14 ++-
 perf-changelog.yaml                           |   4 +-
 3 files changed, 99 insertions(+), 24 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index cc9cc31192..c61eba3284 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -12,15 +12,6 @@ source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-if [[ "$TP" != "8" || "$EP_SIZE" != "1" || "$DP_ATTENTION" != "false" ]]; then
-    echo "Error: GLM-5.2 MI325X full-context qualification requires TP8/EP1 without DP attention" >&2
-    exit 1
-fi
-if [[ "$KV_OFFLOADING" != "none" ]]; then
-    echo "Error: KV_OFFLOADING=$KV_OFFLOADING is not supported by this recipe" >&2
-    exit 1
-fi
-
 if [[ -n "${SLURM_JOB_ID:-}" ]]; then
     echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
 fi
@@ -48,6 +39,21 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    # GLM-5.2's DSA KV pool is replicated across the TP ranks. A 0.75 host
+    # tier adds roughly 0.75M cold-prefix tokens per rank while staying well
+    # inside the MI325X runner's measured 3 TB CPU-DRAM budget.
+    HICACHE_RATIO="${HICACHE_RATIO:-0.75}"
+    CACHE_ARGS=(
+        --enable-hierarchical-cache
+        --hicache-ratio "$HICACHE_RATIO"
+        --hicache-write-policy "${HICACHE_WRITE_POLICY:-write_back}"
+        --hicache-io-backend "${HICACHE_IO_BACKEND:-direct}"
+        --hicache-mem-layout "${HICACHE_MEM_LAYOUT:-page_first_direct}"
+    )
+fi
+
 export PYTHONNOUSERSITE=1
 export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
 export SGLANG_TIMEOUT_KEEP_ALIVE=900
@@ -58,19 +64,61 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900
 export SGLANG_DSA_FUSE_TOPK=false
 export SGLANG_OPT_USE_TOPK_V2=false
 
+USE_SGLANG_ROUTER=false
+SGLANG_BACKEND_PORT="$PORT"
+ROUTER_LOG="$RESULT_DIR/router.log"
+PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE")
+
+# Keep the cookbook profiles as separate topology/cache series so the AgentX
+# dashboard can compare their complete curves instead of overlaying identical
+# labels. All profiles retain the same 1M request limit and 1M HBM KV pool.
+PROFILE=low-latency
+CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 131072)
 MAX_RUNNING_REQUESTS=$((2 * CONC))
-CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS
-[ "$CUDA_GRAPH_MAX_BS" -gt 256 ] && CUDA_GRAPH_MAX_BS=256
+CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$MAX_RUNNING_REQUESTS")
+if [ "$DP_ATTENTION" = "true" ]; then
+    PROFILE=high-throughput
+    USE_SGLANG_ROUTER=true
+    export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
+    SGLANG_BACKEND_PORT=$((PORT + 1))
+    SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
+
+    export SGLANG_SHARED_EXPERT_TP1=1
+    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
+    export SGLANG_DP_USE_GATHERV=1
+    export SGLANG_DP_USE_REDUCE_SCATTER=1
+    export GPU_MAX_HW_QUEUES=5
+
+    PARALLEL_ARGS+=(
+        --dp "$TP"
+        --enable-dp-attention
+        --enable-prefill-delayer
+        --enable-two-batch-overlap
+    )
+    CHUNKED_PREFILL_ARGS=()
+    MAX_RUNNING_REQUESTS=256
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 256)
+elif [ "$EP_SIZE" -gt 1 ]; then
+    PROFILE=balanced
+    CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 32768)
+    MAX_RUNNING_REQUESTS=80
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 128)
+elif [ "$KV_OFFLOADING" != "none" ]; then
+    PROFILE=hicache
+    CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 32768)
+    MAX_RUNNING_REQUESTS=80
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 128)
+fi
+echo "GLM-5.2 MI325X AgentX profile: $PROFILE"
 
 SGLANG_CMD=(
     python3 -m sglang.launch_server
     --model-path "$MODEL_PATH"
     --served-model-name "$MODEL"
     --host 0.0.0.0
-    --port "$PORT"
+    --port "$SGLANG_BACKEND_PORT"
     --trust-remote-code
-    --tp "$TP"
-    --ep-size "$EP_SIZE"
+    "${PARALLEL_ARGS[@]}"
     --dsa-prefill-backend tilelang
     --dsa-decode-backend tilelang
     --dsa-topk-backend torch
@@ -79,10 +127,11 @@ SGLANG_CMD=(
     --reasoning-parser glm45
     --context-length 1048576
     --max-total-tokens 1048576
-    --chunked-prefill-size 131072
+    "${CHUNKED_PREFILL_ARGS[@]}"
     --mem-fraction-static 0.85
     --max-running-requests "$MAX_RUNNING_REQUESTS"
-    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    "${CUDA_GRAPH_ARGS[@]}"
+    "${CACHE_ARGS[@]}"
     --watchdog-timeout 1800
     --enable-metrics
 )
@@ -95,13 +144,33 @@ echo "Starting SGLang server for MI325X..."
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
 
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [ "$USE_SGLANG_ROUTER" = "true" ]; then
+    echo "Starting SGLang router on port $PORT for $TP DP ranks..."
+    python3 -m sglang_router.launch_router \
+        --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
+        --policy consistent_hashing \
+        --request-id-headers x-correlation-id \
+        --dp-aware \
+        --host 0.0.0.0 \
+        --port "$PORT" \
+        --prometheus-host 127.0.0.1 \
+        --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
+        --connect-timeout-secs 900 \
+        --request-timeout-secs 14400 \
+        --disable-health-check \
+        --disable-retries > "$ROUTER_LOG" 2>&1 &
+    ROUTER_PID=$!
+    echo "Router PID: $ROUTER_PID"
+    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
+fi
 
 if [[ "${EVAL_ONLY}" == "true" ]]; then
     export SWEBENCH_AGENT_STEP_LIMIT=150
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
-    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
+    REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 3f05b60a3d..f2081fcf85 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2055,9 +2055,11 @@ minimaxm3-fp8-mi355x-vllm-disagg:
           additional-settings:
           - "DECODE_NODES=1"
 
-# GLM-5.2 FP8 full-context AgentX curve on one MI325X node. The qualification
-# run used about 35% of the explicit 1,048,576-token BF16 KV pool at concurrency
-# 1, so sample densely around the expected concurrency 3-4 capacity knee.
+# GLM-5.2 FP8 full-context AgentX curves on one MI325X node. Every arm retains
+# context-length=1,048,576 and max-total-tokens=1,048,576. The TP8 baseline
+# establishes the low-latency knee; EP8 uses the cookbook balanced batch shape;
+# EP8+DP8 uses its high-throughput shape; HiCache tests whether spilling cold
+# prefixes to the 3 TB host tier restores the cache-hit collapse above conc 3.
 glm5.2-fp8-mi325x-sglang-agentic:
   image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x
   model: zai-org/GLM-5.2-FP8
@@ -2068,8 +2070,12 @@ glm5.2-fp8-mi325x-sglang-agentic:
   multinode: false
   scenarios:
     agentic-coding:
-    - search-space:
+    - dram-utilization: 0.80
+      search-space:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8] }
+      - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8, 10, 12, 16] }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 6, 8, 12, 16, 24, 32], router: { name: sglang-router, version: "0.3.2" } }
+      - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [3, 4, 5, 6, 8, 10, 12, 16] }
 
 # GLM-5.2 FP8 full-context AgentX disaggregation on MI325X. Follow the
 # DeepSeek-V4-Pro AgentX convention: keep each engine/topology to at most four
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 71aa3491bc..d274e1727e 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -4979,8 +4979,8 @@
     - glm5.2-fp8-mi325x-sglang-agentic
     - glm5.2-fp8-mi325x-sglang-disagg-agentic
   description:
-    - "Add a GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context curve at concurrency [1, 2, 3, 4, 5, 6, 8] around the measured KV-capacity knee"
-    - "Run TP8/EP1 STP with an explicit 1,048,576-token context and BF16 GPU KV pool against the complete unfiltered AgentX corpus"
+    - "Add four GLM-5.2 FP8 MI325X SGLang single-node AgentX full-context curves: TP8 low-latency, TP8/EP8 balanced, TP8/EP8/DP8 high-throughput, and TP8 HiCache DRAM offload"
+    - "Keep every arm at an explicit 1,048,576-token request context and 1,048,576-token BF16 GPU KV pool against the complete unfiltered AgentX corpus, with dense concurrency coverage from 1 through 32"
     - "Add TP8 SGLang disaggregated AgentX sweeps for balanced 1P1D, decode-heavy 1P2D, and prefill-heavy 2P1D topologies, with at most four concurrency points per server allocation"
     - "Keep EP and DP attention disabled because MoRI all-to-all remains unstable on the MI325X Broadcom bnxt_re fleet"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2273

From f9cf1f672d4b9797d96106c965ee5ef77f5ea2ac Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Tue, 21 Jul 2026 12:14:03 -0700
Subject: [PATCH 37/42] fix: clean MI325X AgentX server processes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove stale SGLang processes from exclusive eight-GPU allocations, terminate the recipe server on exit, and disable incompatible two-batch overlap for GLM-5.2 DSA.\n\n中文：清理独占八卡分配中的残留 SGLang 进程，在 recipe 退出时终止服务，并为 GLM-5.2 DSA 禁用不兼容的 two-batch overlap。
---
 .../single_node/agentic/glm5.2_fp8_mi325x.sh      | 14 +++++++++++++-
 runners/launch_mi325x-amds.sh                     | 15 +++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index c61eba3284..be21a670cc 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -38,6 +38,19 @@ install_agentic_deps
 
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
+SERVER_PID=""
+ROUTER_PID=""
+
+cleanup() {
+    local pid
+    for pid in "$ROUTER_PID" "$SERVER_PID"; do
+        if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
+            kill -TERM "$pid" 2>/dev/null || true
+            wait "$pid" 2>/dev/null || true
+        fi
+    done
+}
+trap cleanup EXIT
 
 CACHE_ARGS=()
 if require_agentic_kv_offload_backend hicache; then
@@ -93,7 +106,6 @@ if [ "$DP_ATTENTION" = "true" ]; then
         --dp "$TP"
         --enable-dp-attention
         --enable-prefill-delayer
-        --enable-two-batch-overlap
     )
     CHUNKED_PREFILL_ARGS=()
     MAX_RUNNING_REQUESTS=256
diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh
index 663dbd199a..089a833801 100644
--- a/runners/launch_mi325x-amds.sh
+++ b/runners/launch_mi325x-amds.sh
@@ -32,6 +32,21 @@ export TRITON_CACHE_DIR="/tmp/triton-cache-$JOB_ID"
 
 trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
 
+# A terminated enroot step can leave its background SGLang process outside the
+# completed Slurm step. Because each allocation owns all eight GPUs on its node,
+# remove only stale SGLang/router processes before starting the next server.
+# This prevents an old 200-GB/GPU model instance from poisoning later jobs.
+srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c '
+    mapfile -t stale_pids < <(pgrep -f "[s]glang.launch_server|[s]glang_router.launch_router" || true)
+    if [ "${#stale_pids[@]}" -gt 0 ]; then
+        echo "Cleaning stale SGLang processes: ${stale_pids[*]}"
+        kill -TERM "${stale_pids[@]}" 2>/dev/null || sudo -n kill -TERM "${stale_pids[@]}" 2>/dev/null || true
+        sleep 5
+        mapfile -t stale_pids < <(pgrep -f "[s]glang.launch_server|[s]glang_router.launch_router" || true)
+        [ "${#stale_pids[@]}" -eq 0 ] || sudo -n kill -KILL "${stale_pids[@]}" 2>/dev/null || true
+    fi
+'
+
 # Use flock to serialize concurrent imports to the same squash file
 srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c "
     set -euo pipefail

From 7de0ec3e04a3d84d2303c78f56bf003cd1d22af8 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Tue, 21 Jul 2026 15:15:37 -0700
Subject: [PATCH 38/42] fix(amd): run GLM-5.2 DPA profile eagerly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid the gfx942 dynamic shared-memory limit hit by the TileLang DSA kernel during CUDA-graph capture. Other MI325X AgentX profiles continue to use CUDA graphs.\n\n中文：GLM-5.2 DPA 配置改为 eager 模式，避开 TileLang DSA 内核在 CUDA Graph 捕获时超出 gfx942 动态共享内存上限的问题；其他 MI325X AgentX 配置继续使用 CUDA Graph。
---
 benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index be21a670cc..ffe850d694 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -109,7 +109,10 @@ if [ "$DP_ATTENTION" = "true" ]; then
     )
     CHUNKED_PREFILL_ARGS=()
     MAX_RUNNING_REQUESTS=256
-    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 256)
+    # TileLang's DPA DSA kernel needs 115,200 bytes of dynamic shared memory
+    # during graph capture, above gfx942's 65,536-byte limit. Keep the DPA
+    # topology in eager mode; the non-DPA profiles still use CUDA graphs.
+    CUDA_GRAPH_ARGS=(--disable-cuda-graph)
 elif [ "$EP_SIZE" -gt 1 ]; then
     PROFILE=balanced
     CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 32768)

From b97e649a7f698a11703f08d6829cd53d4dfd478f Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Tue, 21 Jul 2026 16:27:17 -0700
Subject: [PATCH 39/42] fix(amd): use AITER DSA for GLM-5.2 DPA
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Route only the MI325X DP-attention profile through SGLang's AMD-specific AITER DSA backend. TileLang remains enabled for the other profiles, while DPA restores CUDA graphs for a representative throughput curve.\n\n中文：仅将 MI325X 的 DP attention 配置切换到 SGLang 面向 AMD 的 AITER DSA 后端。其他配置继续使用 TileLang，DPA 同时恢复 CUDA Graph，以获得具有代表性的吞吐量曲线。
---
 .../single_node/agentic/glm5.2_fp8_mi325x.sh     | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index ffe850d694..3977274f2e 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -89,6 +89,8 @@ PROFILE=low-latency
 CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 131072)
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$MAX_RUNNING_REQUESTS")
+DSA_PREFILL_BACKEND=tilelang
+DSA_DECODE_BACKEND=tilelang
 if [ "$DP_ATTENTION" = "true" ]; then
     PROFILE=high-throughput
     USE_SGLANG_ROUTER=true
@@ -109,10 +111,12 @@ if [ "$DP_ATTENTION" = "true" ]; then
     )
     CHUNKED_PREFILL_ARGS=()
     MAX_RUNNING_REQUESTS=256
-    # TileLang's DPA DSA kernel needs 115,200 bytes of dynamic shared memory
-    # during graph capture, above gfx942's 65,536-byte limit. Keep the DPA
-    # topology in eager mode; the non-DPA profiles still use CUDA graphs.
-    CUDA_GRAPH_ARGS=(--disable-cuda-graph)
+    # TileLang's DPA DSA kernel needs 115,200 bytes of dynamic shared memory,
+    # above gfx942's 65,536-byte per-block limit even in eager mode. AITER is
+    # SGLang's alternate ROCm DSA backend and avoids that kernel.
+    DSA_PREFILL_BACKEND=aiter
+    DSA_DECODE_BACKEND=aiter
+    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 256)
 elif [ "$EP_SIZE" -gt 1 ]; then
     PROFILE=balanced
     CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 32768)
@@ -134,8 +138,8 @@ SGLANG_CMD=(
     --port "$SGLANG_BACKEND_PORT"
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
-    --dsa-prefill-backend tilelang
-    --dsa-decode-backend tilelang
+    --dsa-prefill-backend "$DSA_PREFILL_BACKEND"
+    --dsa-decode-backend "$DSA_DECODE_BACKEND"
     --dsa-topk-backend torch
     --kv-cache-dtype bfloat16
     --tool-call-parser glm47

From f4d007b7f2cfe605615006b78f91b52c42a0a029 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Tue, 21 Jul 2026 21:03:56 -0700
Subject: [PATCH 40/42] fix(amd): exclude unsupported GLM-5.2 DPA sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove the MI325X DPA arm after both available ROCm DSA backends failed: TileLang exceeds gfx942 shared memory and AITER lacks the required decode kernel. Keep the complete TP8, EP8, and HiCache AgentX curves.\n\n中文：移除 MI325X DPA 扫描配置，因为两个可用的 ROCm DSA 后端均无法运行：TileLang 超出 gfx942 共享内存上限，AITER 缺少所需的解码内核。保留完整的 TP8、EP8 和 HiCache AgentX 曲线。
---
 .../single_node/agentic/glm5.2_fp8_mi325x.sh  | 75 +++----------------
 configs/amd-master.yaml                       |  7 +-
 2 files changed, 14 insertions(+), 68 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index 3977274f2e..e01f9a18f6 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -39,16 +39,12 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 SERVER_PID=""
-ROUTER_PID=""
 
 cleanup() {
-    local pid
-    for pid in "$ROUTER_PID" "$SERVER_PID"; do
-        if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
-            kill -TERM "$pid" 2>/dev/null || true
-            wait "$pid" 2>/dev/null || true
-        fi
-    done
+    if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
+        kill -TERM "$SERVER_PID" 2>/dev/null || true
+        wait "$SERVER_PID" 2>/dev/null || true
+    fi
 }
 trap cleanup EXIT
 
@@ -77,9 +73,6 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900
 export SGLANG_DSA_FUSE_TOPK=false
 export SGLANG_OPT_USE_TOPK_V2=false
 
-USE_SGLANG_ROUTER=false
-SGLANG_BACKEND_PORT="$PORT"
-ROUTER_LOG="$RESULT_DIR/router.log"
 PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE")
 
 # Keep the cookbook profiles as separate topology/cache series so the AgentX
@@ -89,35 +82,7 @@ PROFILE=low-latency
 CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 131072)
 MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$MAX_RUNNING_REQUESTS")
-DSA_PREFILL_BACKEND=tilelang
-DSA_DECODE_BACKEND=tilelang
-if [ "$DP_ATTENTION" = "true" ]; then
-    PROFILE=high-throughput
-    USE_SGLANG_ROUTER=true
-    export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true
-    SGLANG_BACKEND_PORT=$((PORT + 1))
-    SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000))
-
-    export SGLANG_SHARED_EXPERT_TP1=1
-    export SGLANG_DP_SHARED_EXPERT_LOCAL=1
-    export SGLANG_DP_USE_GATHERV=1
-    export SGLANG_DP_USE_REDUCE_SCATTER=1
-    export GPU_MAX_HW_QUEUES=5
-
-    PARALLEL_ARGS+=(
-        --dp "$TP"
-        --enable-dp-attention
-        --enable-prefill-delayer
-    )
-    CHUNKED_PREFILL_ARGS=()
-    MAX_RUNNING_REQUESTS=256
-    # TileLang's DPA DSA kernel needs 115,200 bytes of dynamic shared memory,
-    # above gfx942's 65,536-byte per-block limit even in eager mode. AITER is
-    # SGLang's alternate ROCm DSA backend and avoids that kernel.
-    DSA_PREFILL_BACKEND=aiter
-    DSA_DECODE_BACKEND=aiter
-    CUDA_GRAPH_ARGS=(--cuda-graph-max-bs 256)
-elif [ "$EP_SIZE" -gt 1 ]; then
+if [ "$EP_SIZE" -gt 1 ]; then
     PROFILE=balanced
     CHUNKED_PREFILL_ARGS=(--chunked-prefill-size 32768)
     MAX_RUNNING_REQUESTS=80
@@ -135,11 +100,11 @@ SGLANG_CMD=(
     --model-path "$MODEL_PATH"
     --served-model-name "$MODEL"
     --host 0.0.0.0
-    --port "$SGLANG_BACKEND_PORT"
+    --port "$PORT"
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
-    --dsa-prefill-backend "$DSA_PREFILL_BACKEND"
-    --dsa-decode-backend "$DSA_DECODE_BACKEND"
+    --dsa-prefill-backend tilelang
+    --dsa-decode-backend tilelang
     --dsa-topk-backend torch
     --kv-cache-dtype bfloat16
     --tool-call-parser glm47
@@ -163,33 +128,13 @@ echo "Starting SGLang server for MI325X..."
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
 
-wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "$USE_SGLANG_ROUTER" = "true" ]; then
-    echo "Starting SGLang router on port $PORT for $TP DP ranks..."
-    python3 -m sglang_router.launch_router \
-        --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \
-        --policy consistent_hashing \
-        --request-id-headers x-correlation-id \
-        --dp-aware \
-        --host 0.0.0.0 \
-        --port "$PORT" \
-        --prometheus-host 127.0.0.1 \
-        --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \
-        --connect-timeout-secs 900 \
-        --request-timeout-secs 14400 \
-        --disable-health-check \
-        --disable-retries > "$ROUTER_LOG" 2>&1 &
-    ROUTER_PID=$!
-    echo "Router PID: $ROUTER_PID"
-    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
 
 if [[ "${EVAL_ONLY}" == "true" ]]; then
     export SWEBENCH_AGENT_STEP_LIMIT=150
     run_eval --port "$PORT"
 else
     build_replay_cmd "$RESULT_DIR"
-    REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
     run_agentic_replay_and_write_outputs "$RESULT_DIR"
 fi
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index f2081fcf85..ce702fb9ed 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -2058,8 +2058,10 @@ minimaxm3-fp8-mi355x-vllm-disagg:
 # GLM-5.2 FP8 full-context AgentX curves on one MI325X node. Every arm retains
 # context-length=1,048,576 and max-total-tokens=1,048,576. The TP8 baseline
 # establishes the low-latency knee; EP8 uses the cookbook balanced batch shape;
-# EP8+DP8 uses its high-throughput shape; HiCache tests whether spilling cold
-# prefixes to the 3 TB host tier restores the cache-hit collapse above conc 3.
+# HiCache tests whether spilling cold prefixes to the 3 TB host tier restores
+# the cache-hit collapse above conc 3. DP attention is excluded: TileLang's
+# DSA kernel exceeds gfx942's per-block shared-memory limit, while AITER has no
+# decode kernel for GLM-5.2's bf16 GQA-64 geometry in the current image.
 glm5.2-fp8-mi325x-sglang-agentic:
   image: lmsysorg/sglang:v0.5.15.post1-rocm720-mi30x
   model: zai-org/GLM-5.2-FP8
@@ -2074,7 +2076,6 @@ glm5.2-fp8-mi325x-sglang-agentic:
       search-space:
       - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8] }
       - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 8, 10, 12, 16] }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [4, 6, 8, 12, 16, 24, 32], router: { name: sglang-router, version: "0.3.2" } }
       - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [3, 4, 5, 6, 8, 10, 12, 16] }
 
 # GLM-5.2 FP8 full-context AgentX disaggregation on MI325X. Follow the

From 6cd533073b91825b84710f1f430a8ab6fe03a387 Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Wed, 22 Jul 2026 00:27:59 -0700
Subject: [PATCH 41/42] fix(amd): clean GLM-5.2 scheduler process groups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Launch the MI325X SGLang server in a dedicated process group and terminate the full group on exit. Extend exclusive-node cleanup to remove orphaned sglang::scheduler children that retained about 200 GB per GPU and poisoned subsequent sweep jobs.\n\n中文：将 MI325X SGLang 服务放入独立进程组，并在退出时终止整个进程组。同时扩展独占节点清理逻辑，移除残留的 sglang::scheduler 子进程；这些进程会持续占用每张 GPU 约 200 GB 显存，导致后续扫描任务失败。
---
 benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh | 12 +++++++++---
 runners/launch_mi325x-amds.sh                       |  5 +++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
index e01f9a18f6..4fc33b12fc 100755
--- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
+++ b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x.sh
@@ -39,12 +39,15 @@ install_agentic_deps
 SERVER_LOG="$RESULT_DIR/server.log"
 mkdir -p "$RESULT_DIR"
 SERVER_PID=""
+SERVER_PGID=""
 
 cleanup() {
-    if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
+    if [[ -n "$SERVER_PGID" ]]; then
+        kill -TERM -- "-$SERVER_PGID" 2>/dev/null || true
+    elif [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
         kill -TERM "$SERVER_PID" 2>/dev/null || true
-        wait "$SERVER_PID" 2>/dev/null || true
     fi
+    [[ -z "$SERVER_PID" ]] || wait "$SERVER_PID" 2>/dev/null || true
 }
 trap cleanup EXIT
 
@@ -124,8 +127,11 @@ printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
 printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
 
 echo "Starting SGLang server for MI325X..."
-"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+# Keep the launcher and its renamed sglang::scheduler TP children in one
+# process group so teardown cannot leave a 200-GB/GPU child behind.
+setsid "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
+SERVER_PGID=$SERVER_PID
 echo "Server PID: $SERVER_PID"
 
 wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh
index 089a833801..cdaf03b970 100644
--- a/runners/launch_mi325x-amds.sh
+++ b/runners/launch_mi325x-amds.sh
@@ -37,12 +37,13 @@ trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
 # remove only stale SGLang/router processes before starting the next server.
 # This prevents an old 200-GB/GPU model instance from poisoning later jobs.
 srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c '
-    mapfile -t stale_pids < <(pgrep -f "[s]glang.launch_server|[s]glang_router.launch_router" || true)
+    stale_pattern="[s]glang.launch_server|[s]glang_router.launch_router|[s]glang::"
+    mapfile -t stale_pids < <(pgrep -f "$stale_pattern" || true)
     if [ "${#stale_pids[@]}" -gt 0 ]; then
         echo "Cleaning stale SGLang processes: ${stale_pids[*]}"
         kill -TERM "${stale_pids[@]}" 2>/dev/null || sudo -n kill -TERM "${stale_pids[@]}" 2>/dev/null || true
         sleep 5
-        mapfile -t stale_pids < <(pgrep -f "[s]glang.launch_server|[s]glang_router.launch_router" || true)
+        mapfile -t stale_pids < <(pgrep -f "$stale_pattern" || true)
         [ "${#stale_pids[@]}" -eq 0 ] || sudo -n kill -KILL "${stale_pids[@]}" 2>/dev/null || true
     fi
 '

From 08b0877886ef9fa13d10b3fdf593b82d4a5504bf Mon Sep 17 00:00:00 2001
From: Jordan Nanos <jordan.nanos@gmail.com>
Date: Fri, 24 Jul 2026 08:14:35 -0700
Subject: [PATCH 42/42] fix(amd): reclaim MI325X AgentX runner disk
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Clean job-scoped XDG and Triton caches when each Slurm allocation exits, and remove MI325X benchmark results after artifact upload so repeated full-context sweeps do not exhaust runner storage.

中文：在每个 Slurm 分配退出时清理该任务专属的 XDG 与 Triton 缓存，并在产物上传后删除 MI325X 基准测试结果目录，避免重复执行全上下文扫描时耗尽 runner 存储空间。
---
 .github/workflows/benchmark-tmpl.yml |  8 ++++++++
 runners/launch_mi325x-amds.sh        | 15 ++++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml
index 3b2614e145..8240d00d68 100644
--- a/.github/workflows/benchmark-tmpl.yml
+++ b/.github/workflows/benchmark-tmpl.yml
@@ -202,6 +202,14 @@ jobs:
             done
           fi
 
+          # MI325X AgentX artifacts can occupy hundreds of MB per job. They have
+          # already been uploaded when this anchor runs post-job; removing the
+          # prior workspace copy pre-job also recovers interrupted-run output.
+          if [[ "$RUNNER_NAME" == mi325x-amds_* ]]; then
+            echo "[MI325X] Cleaning benchmark workspace results ..."
+            rm -rf -- "$GITHUB_WORKSPACE/results"
+          fi
+
           # Cleanup SLURM resources
           if command -v squeue >/dev/null 2>&1; then
             echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} ..."
diff --git a/runners/launch_mi325x-amds.sh b/runners/launch_mi325x-amds.sh
index cdaf03b970..30747b5e70 100644
--- a/runners/launch_mi325x-amds.sh
+++ b/runners/launch_mi325x-amds.sh
@@ -30,7 +30,20 @@ export PORT=$(( 40000 + (JOB_ID % 10000) ))
 export XDG_CACHE_HOME="/tmp/xdg-cache-$JOB_ID"
 export TRITON_CACHE_DIR="/tmp/triton-cache-$JOB_ID"
 
-trap 'rc=$?; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT
+cleanup_allocation() {
+    local rc=$?
+
+    # AgentX restores a multi-GB mmap dataset under XDG_CACHE_HOME. Remove this
+    # allocation's cache and compiled kernels so repeated sweeps cannot fill the
+    # worker's local disk. The paths include the resolved Slurm job ID and are
+    # therefore isolated from other allocations.
+    srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" \
+        bash -c 'rm -rf -- "$XDG_CACHE_HOME" "$TRITON_CACHE_DIR"' \
+        2>/dev/null || true
+    scancel "$JOB_ID" 2>/dev/null || true
+    exit "$rc"
+}
+trap cleanup_allocation EXIT
 
 # A terminated enroot step can leave its background SGLang process outside the
 # completed Slurm step. Because each allocation owns all eight GPUs on its node,