From 4c8784b0bb82060cffe2378994fb35c3932db942 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:06:10 -0400 Subject: [PATCH 01/12] feat: add GLM-5.2 H200 full-context AgentX recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a three-node Dynamo SGLang qualification path with PCP8 DSA LayerSplit prefill, TP16 decode, Mooncake transfer, HiCache, and the unfiltered 1M-context AgentX trace. DCP remains disabled because SGLang DSA decode-context parallelism is not merged; TP16 supplies the required HBM headroom without mislabeling unsupported functionality. 中文：新增三节点 Dynamo SGLang 资格验证路径：预填充采用 PCP8 DSA LayerSplit，解码采用 TP16，并启用 Mooncake 传输、HiCache 和未截断的 1M 上下文 AgentX 轨迹。 由于 SGLang 的 DSA 解码上下文并行尚未合并，本配置不启用 DCP；改用 TP16 提供所需显存余量，避免将未支持的能力标记为可用。 --- benchmarks/benchmark_lib.sh | 2 +- .../disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 152 ++++++++++++++++++ configs/nvidia-master.yaml | 39 +++++ runners/launch_h200-dgxc-slurm.sh | 62 ++++++- 4 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index b4c947932b..96492072c8 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1660,7 +1660,7 @@ resolve_trace_source() { # WEKA_LOADER_OVERRIDE. local default_loader case "${MODEL_PREFIX:-}" in - dsv4*|minimaxm3*) + dsv4*|glm5.2*|minimaxm3*) default_loader="semianalysis_cc_traces_weka_062126" ;; *) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml new file mode 100644 index 0000000000..fb042e452a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -0,0 +1,152 @@ +name: "disagg-h200-1p1d-pcp8-tp16-hicache" + +# Full-context GLM-5.2 AgentX qualification for three 8xH200 nodes. +# +# Prefill uses SGLang's DSA LayerSplit path: TP8 supplies eight context- +# parallel ranks (effective attention TP1 x PCP8), which shard KV/indexer +# cache layers before Mooncake transfers complete cache shards to decode. +# Decode uses TP16 across two nodes because DSA DCP is not available in this +# SGLang build; TP16 leaves enough HBM for the explicit 1M-token FP8 KV pool. +# STP is deliberate until the upstream GLM-5.2 MTP + HiCache corruption issue +# is resolved. + +model: + path: "glm5.2-fp8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8" + precision: "fp8" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: h200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + + prefill_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + + decode_environment: + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + + sglang_config: + prefill: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + + decode: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 16 + ep-size: 1 + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: fp8_e4m3 + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 32 + cuda-graph-max-bs: 32 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 1 + hicache-io-backend: direct + +sbatch_directives: + mem: "0" + cpus-per-task: "144" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 7d7af13236..219cf31ba6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9830,6 +9830,45 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 16 ep: 16 dp-attn: true + +# GLM-5.2 FP8 full-context AgentX qualification on H200. SGLang prefill CP +# runs inside one TP8 worker, so the matrix describes the effective attention +# topology as TP1 x PCP8 while the recipe launches eight ranks. LayerSplit +# shards DSA KV/indexer cache layers across those ranks. Decode remains TP16: +# upstream DSA decode-context parallelism is not merged, and two H200 nodes +# are required to leave runtime headroom beside the explicit 1M FP8 KV pool. +glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: + image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + model: zai-org/GLM-5.2-FP8 + model-prefix: glm5.2 + runner: cluster:h200-dgxc + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev1" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: none + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..8ce31494cb 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -9,6 +9,15 @@ set -x if [[ "$IS_MULTINODE" == "true" ]]; then + # Resolve InferenceX-owned recipes before entering the cloned srt-slurm + # checkout. AgentX also needs persistent host caches on every worker. + LOCAL_CONFIG_FILE="" + if [[ -n "${CONFIG_FILE:-}" && -f "$GITHUB_WORKSPACE/$CONFIG_FILE" ]]; then + LOCAL_CONFIG_FILE="$GITHUB_WORKSPACE/$CONFIG_FILE" + fi + AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" + HF_HUB_CACHE_HOST_PATH="/models/gharunners/hf-hub-cache" + # MODEL_PATH: Override with pre-downloaded paths on H200 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H200 cluster. @@ -16,6 +25,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" + elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="${GLM52_FP8_MODEL_PATH:-/models/GLM-5.2-FP8}" + if [[ ! -d "$MODEL_PATH" ]]; then + export MODEL_PATH="hf:zai-org/GLM-5.2-FP8" + fi + export SRT_SLURM_MODEL_PREFIX="glm5.2-fp8" else echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" exit 1 @@ -42,7 +57,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then + # v1.0.10 contains the custom benchmark hook, session-affinity + # frontend, and large-request nginx fix used by SGLang AgentX recipes. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.10 + elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -51,6 +72,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout sa-submission-q2-2026 fi + if [[ -n "$LOCAL_CONFIG_FILE" ]]; then + mkdir -p "$(dirname "$CONFIG_FILE")" + cp "$LOCAL_CONFIG_FILE" "$CONFIG_FILE" + fi + echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh source $HOME/.local/bin/env @@ -71,7 +97,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $FRAMEWORK == "dynamo-sglang" ]]; then # SGLang container mapping - SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + if [[ $MODEL_PREFIX == "glm5.2" ]]; then + SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + else + SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + fi CONTAINER_KEY="$IMAGE" elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) @@ -79,6 +109,24 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" fi + if ! unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + DOCKER_IMAGE=$(echo "$IMAGE" | sed 's/#/\//g') + LOCK_FILE="${SQUASH_FILE}.lock" + srun --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --nodes=1 --ntasks=1 --time=30 --job-name="$RUNNER_NAME" \ + bash -c " + exec 9>\"$LOCK_FILE\" + flock -w 1800 9 || { echo 'Failed to acquire $LOCK_FILE'; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" >/dev/null 2>&1; then + exit 0 + fi + rm -f \"$SQUASH_FILE\" + export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot + mkdir -p \"\$ENROOT_CACHE_PATH\" + enroot import -o \"$SQUASH_FILE\" docker://$DOCKER_IMAGE + " + fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -98,6 +146,10 @@ gpus_per_node: 8 network_interface: "" # Path to srtctl repo root (where the configs live) srtctl_root: "${SRTCTL_ROOT}" +# Persistent AgentX dataset caches mounted into every worker container. +default_mounts: + "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" + "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" # Model path aliases model_paths: "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}" @@ -135,7 +187,11 @@ EOF sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "${CONFIG_FILE%%:*}" printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + WORKLOAD_TAG="${ISL}x${OSL}" + if [[ "$IS_AGENTIC" == "1" ]]; then + WORKLOAD_TAG="agentic" + fi + SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${WORKLOAD_TAG},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output From f12e28559aaa6c3f30853279670e5ddeb11990b3 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:06:47 -0400 Subject: [PATCH 02/12] chore: register GLM-5.2 H200 AgentX sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append PR #2275 to the performance changelog so the full-context PCP and HiCache qualification is discoverable by sweep setup. 中文：将 PR #2275 追加到性能变更日志，使扫描流程能够发现全上下文 PCP 与 HiCache 资格验证配置。 --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b358f10bf1..8ad1e9cadf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4933,3 +4933,11 @@ description: - "Add MiniMax M3 NVFP4 B300 Dynamo-vLLM disaggregated EAGLE3 recipes" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2182 + +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache + description: + - "Add GLM-5.2 FP8 full-context H200 AgentX qualification with SGLang DSA prefill context parallelism (TP1 x PCP8), LayerSplit, and TP16 decode" + - "Enable Mooncake PD transfer and HiCache on prefill and decode while replaying the unfiltered 1M-context AgentX trace at concurrency 1" + - "Use STP because DSA decode context parallelism is not merged and the GLM-5.2 MTP plus HiCache correctness issue remains open" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2275 From b20b9b0c10a1b802c0fde4721874bd81cb3b86a8 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:27:41 -0400 Subject: [PATCH 03/12] fix: preserve H200 AgentX recipe after srt setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overlay the checked-in recipe after make setup refreshes the srt-slurm recipe tree. This fixes the missing CONFIG_FILE failure observed in unofficial run 29657445141. 中文：在 make setup 刷新 srt-slurm 配方目录后再覆盖仓库内配方，修复非正式运行 29657445141 中 CONFIG_FILE 丢失的问题。 --- runners/launch_h200-dgxc-slurm.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 8ce31494cb..c267b4d9f1 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -72,11 +72,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout sa-submission-q2-2026 fi - if [[ -n "$LOCAL_CONFIG_FILE" ]]; then - mkdir -p "$(dirname "$CONFIG_FILE")" - cp "$LOCAL_CONFIG_FILE" "$CONFIG_FILE" - fi - echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh source $HOME/.local/bin/env @@ -172,6 +167,13 @@ EOF echo "Running make setup..." make setup ARCH=x86_64 + # make setup refreshes the srt-slurm recipe tree. Overlay the checked-in + # InferenceX recipe afterwards so it remains available to srtctl apply. + if [[ -n "$LOCAL_CONFIG_FILE" ]]; then + mkdir -p "$(dirname "$CONFIG_FILE")" + cp "$LOCAL_CONFIG_FILE" "$CONFIG_FILE" + fi + # Export eval-related env vars for srt-slurm post-benchmark eval export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" From 7c138499bc5b36b3cbd89557b1f3c080e167e69a Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:30:10 -0400 Subject: [PATCH 04/12] fix: map checked-in H200 recipes into srt-slurm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve repository-owned recipe paths under benchmarks/multi_node/srt-slurm-recipes and copy them into the generated recipes tree after setup. Preserve selector-aware CONFIG_FILE handling through a normalized CONFIG_PATH. 中文：将仓库内的 H200 配方从 benchmarks/multi_node/srt-slurm-recipes 正确映射到 srt-slurm 生成的 recipes 目录，并在 setup 后覆盖写入；同时通过标准化的 CONFIG_PATH 保留对选择器后缀的兼容。 --- runners/launch_h200-dgxc-slurm.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index c267b4d9f1..c23fabb947 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -11,9 +11,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # Resolve InferenceX-owned recipes before entering the cloned srt-slurm # checkout. AgentX also needs persistent host caches on every worker. + CONFIG_PATH="${CONFIG_FILE%%:*}" LOCAL_CONFIG_FILE="" - if [[ -n "${CONFIG_FILE:-}" && -f "$GITHUB_WORKSPACE/$CONFIG_FILE" ]]; then - LOCAL_CONFIG_FILE="$GITHUB_WORKSPACE/$CONFIG_FILE" + LOCAL_CONFIG_PATH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" + if [[ -n "${CONFIG_FILE:-}" && -f "$LOCAL_CONFIG_PATH" ]]; then + LOCAL_CONFIG_FILE="$LOCAL_CONFIG_PATH" fi AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" HF_HUB_CACHE_HOST_PATH="/models/gharunners/hf-hub-cache" @@ -170,8 +172,8 @@ EOF # make setup refreshes the srt-slurm recipe tree. Overlay the checked-in # InferenceX recipe afterwards so it remains available to srtctl apply. if [[ -n "$LOCAL_CONFIG_FILE" ]]; then - mkdir -p "$(dirname "$CONFIG_FILE")" - cp "$LOCAL_CONFIG_FILE" "$CONFIG_FILE" + mkdir -p "$(dirname "$CONFIG_PATH")" + cp "$LOCAL_CONFIG_FILE" "$CONFIG_PATH" fi # Export eval-related env vars for srt-slurm post-benchmark eval @@ -186,9 +188,9 @@ EOF fi # Override the job name in the config file with the runner name - sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "${CONFIG_FILE%%:*}" - printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "${CONFIG_FILE%%:*}" + sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH" + printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH" WORKLOAD_TAG="${ISL}x${OSL}" if [[ "$IS_AGENTIC" == "1" ]]; then WORKLOAD_TAG="agentic" From 433042d5232ae10864014c8c999c06273dd09702 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:32:58 -0400 Subject: [PATCH 05/12] fix: load GLM-5.2 through the shared HF cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the native srt-slurm hf: model path when the H200 cluster has no staged GLM-5.2 directory. Point both SGLang worker roles at the persistent shared Hugging Face cache. 中文：当 H200 集群尚未预置 GLM-5.2 模型目录时，改用 srt-slurm 原生支持的 hf: 模型路径，并让 SGLang 预填充与解码节点共同使用持久化的 Hugging Face 共享缓存。 --- .../glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index fb042e452a..75ccd869aa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -11,7 +11,7 @@ name: "disagg-h200-1p1d-pcp8-tp16-hicache" # is resolved. model: - path: "glm5.2-fp8" + path: "hf:zai-org/GLM-5.2-FP8" container: "lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8" precision: "fp8" @@ -58,6 +58,7 @@ backend: type: sglang prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" @@ -67,6 +68,7 @@ backend: SGLANG_TIMEOUT_KEEP_ALIVE: "900" decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" From 89223d13a54994acc2a1c492e16fee8619d706e3 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:35:07 -0400 Subject: [PATCH 06/12] fix: use native H200 CPU allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the GB300-derived 144-CPU batch override that H200 Slurm nodes cannot satisfy. Retain full-node memory allocation for batch and worker steps. 中文：移除源自 GB300、H200 Slurm 节点无法满足的 144 CPU 批处理覆盖配置，同时保留批处理与工作节点步骤的整节点内存分配。 --- .../glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index 75ccd869aa..8613720dab 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -134,7 +134,6 @@ backend: sbatch_directives: mem: "0" - cpus-per-task: "144" srun_options: mem: "0" From d957349cbc68dfbff2256e9878b1f29f71e29f29 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 15:39:59 -0400 Subject: [PATCH 07/12] fix: keep radix cache enabled for HiCache decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the SGLang PD decode radix-cache override so hierarchical cache does not conflict with the default chunk-cache mode. Declare HF_HOME alongside HF_HUB_CACHE for the shared model cache. 中文：启用 SGLang PD 解码侧的 radix cache 覆盖配置，避免分层缓存与默认 chunk cache 模式冲突；同时为共享模型缓存补充 HF_HOME。 --- .../glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index 8613720dab..a70154f75a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -58,6 +58,7 @@ backend: type: sglang prefill_environment: + HF_HOME: "/hf_hub_cache" HF_HUB_CACHE: "/hf_hub_cache" PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" @@ -68,6 +69,7 @@ backend: SGLANG_TIMEOUT_KEEP_ALIVE: "900" decode_environment: + HF_HOME: "/hf_hub_cache" HF_HUB_CACHE: "/hf_hub_cache" PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" @@ -118,6 +120,7 @@ backend: ep-size: 1 disaggregation-transfer-backend: mooncake disaggregation-mode: decode + disaggregation-decode-enable-radix-cache: true kv-cache-dtype: fp8_e4m3 context-length: 1048576 max-total-tokens: 1048576 From c48a286b4453a931f561486c713cc7f18d4b7690 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 16:17:58 -0400 Subject: [PATCH 08/12] fix: use HiCache-compatible radix trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unified radix-tree override from both GLM-5.2 workers. The pinned SGLang decode HiCache path requires query_storage_hit_length, which UnifiedRadixCache does not provide. 中文：为 GLM-5.2 的预填充和解码 worker 移除统一基数树覆盖配置。当前固定版本的 SGLang 解码 HiCache 路径需要 query_storage_hit_length，而 UnifiedRadixCache 未实现该接口。 --- .../glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index a70154f75a..b00bf061ad 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -63,7 +63,6 @@ backend: PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_USE_TOPK_V2: "1" SGLANG_TIMEOUT_KEEP_ALIVE: "900" @@ -74,7 +73,6 @@ backend: PIP_BREAK_SYSTEM_PACKAGES: "1" PYTHONUNBUFFERED: "1" SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_ENABLE_UNIFIED_RADIX_TREE: "1" SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_USE_TOPK_V2: "1" SGLANG_TIMEOUT_KEEP_ALIVE: "900" From e269af4dd56b6a088aac0e991661beb1d94e1b7c Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sat, 18 Jul 2026 16:58:42 -0400 Subject: [PATCH 09/12] fix: use layer-first HiCache for PCP prefill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the prefill HiCache host layout to layer_first. SGLang requires this layout when direct HiCache backup is combined with layer-sharded PCP cache. 中文：将预填充 HiCache 的主机内存布局设为 layer_first。SGLang 在 direct HiCache 备份与 PCP 分层分片缓存组合使用时要求该布局。 --- .../glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index b00bf061ad..db24a4d6ca 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -106,6 +106,7 @@ backend: hicache-write-policy: write_back hicache-ratio: 1 hicache-io-backend: direct + hicache-mem-layout: layer_first decode: host: 0.0.0.0 From bc7bde76d6b0f0ebd3d1f8ecd1e1e0803183c840 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sun, 19 Jul 2026 10:45:29 -0400 Subject: [PATCH 10/12] feat: expand GLM-5.2 H200 AgentX curve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add seven concurrency points and split them across two allocations so each AgentX run remains within the eight-hour Slurm limit.\n\n中文：将并发度扩展为七个数据点，并拆分至两个资源分配任务，确保每次 AgentX 基准测试均能在八小时 Slurm 时限内完成。 --- configs/nvidia-master.yaml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 219cf31ba6..86c9a6fa8f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9853,7 +9853,26 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: none - conc-list: [1] + # Keep each allocation to at most four one-hour AgentX points so the + # benchmark and trajectory drain fit within the recipe's 8-hour limit. + conc-list: [1, 2, 3, 4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [5, 6, 8] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 078265956a51ef3d37df93e894571cc4d836cd87 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Sun, 19 Jul 2026 14:17:05 -0400 Subject: [PATCH 11/12] fix: isolate H200 AgentX curve points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep HiCache on PCP prefill while using decode RadixCache, matching the stable disaggregated pattern and avoiding cross-rank HiCache restore failures. Run each concurrency point in its own allocation so failures do not mask later results.\n\n中文：在 PCP 预填充侧保留 HiCache，解码侧改用 RadixCache，与稳定的分离式推理方案保持一致，避免 HiCache 恢复长前缀时出现跨 rank 集合通信错误。每个并发度使用独立资源分配任务，防止单点故障掩盖后续结果。 --- .../disagg-h200-1p1d-pcp8-tp16-hicache.yaml | 4 - configs/nvidia-master.yaml | 93 ++++++++++++++++++- 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml index db24a4d6ca..91ffe94167 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml @@ -129,10 +129,6 @@ backend: watchdog-timeout: 86400 stream-interval: 60 enable-metrics: true - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct sbatch_directives: mem: "0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 86c9a6fa8f..2239423c6b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9853,9 +9853,26 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: none - # Keep each allocation to at most four one-hour AgentX points so the - # benchmark and trajectory drain fit within the recipe's 8-hour limit. - conc-list: [1, 2, 3, 4] + # Isolate each point: long-prefix restore failures at one concurrency + # must not discard completed points or prevent later points from running. + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [2] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: @@ -9872,7 +9889,75 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: ep: 1 dp-attn: false - spec-decoding: none - conc-list: [5, 6, 8] + conc-list: [3] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [4] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [5] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [6] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + - spec-decoding: none + conc-list: [8] kv-offloading: dram kv-offload-backend: { name: hicache } prefill: From 7867aedf165cc4fc6e8d820537cc74826b2ae827 Mon Sep 17 00:00:00 2001 From: Jordan Nanos Date: Tue, 21 Jul 2026 12:21:01 -0700 Subject: [PATCH 12/12] feat: replace H200 AgentX HiCache with HiSparse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the TP16 HiCache topology with the validated PCP8 prefill and TP8/DPA HiSparse decode topology. Register the complete c1-c8 AgentX curve and record successful run 29820102138. 中文：将 TP16 HiCache 拓扑替换为已验证的 PCP8 预填充与 TP8/DPA HiSparse 解码拓扑。注册完整的 c1-c8 AgentX 扫描，并记录成功运行 29820102138。 --- ...sagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml} | 57 +++----- configs/nvidia-master.yaml | 124 ++---------------- perf-changelog.yaml | 10 ++ 3 files changed, 36 insertions(+), 155 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/{disagg-h200-1p1d-pcp8-tp16-hicache.yaml => disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml} (66%) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml similarity index 66% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml index 91ffe94167..97c5d0ceeb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml @@ -1,14 +1,7 @@ -name: "disagg-h200-1p1d-pcp8-tp16-hicache" +name: "disagg-h200-1p1d-pcp8-tp8-dp8-hisparse" -# Full-context GLM-5.2 AgentX qualification for three 8xH200 nodes. -# -# Prefill uses SGLang's DSA LayerSplit path: TP8 supplies eight context- -# parallel ranks (effective attention TP1 x PCP8), which shard KV/indexer -# cache layers before Mooncake transfers complete cache shards to decode. -# Decode uses TP16 across two nodes because DSA DCP is not available in this -# SGLang build; TP16 leaves enough HBM for the explicit 1M-token FP8 KV pool. -# STP is deliberate until the upstream GLM-5.2 MTP + HiCache corruption issue -# is resolved. +# Experimental HiSparse decode. This is not HiCache: full KV lives in pinned +# host memory while a fixed sparse working set remains in HBM. model: path: "hf:zai-org/GLM-5.2-FP8" @@ -22,19 +15,15 @@ dynamo: slurm: time_limit: "8:00:00" -health_check: - max_attempts: 1440 - interval_seconds: 10 - resources: gpu_type: h200 gpus_per_node: 8 prefill_nodes: 1 - decode_nodes: 2 + decode_nodes: 1 prefill_workers: 1 decode_workers: 1 gpus_per_prefill: 8 - gpus_per_decode: 16 + gpus_per_decode: 8 infra: etcd_nats_dedicated_node: true @@ -56,18 +45,7 @@ frontend: backend: type: sglang - - prefill_environment: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache" - PIP_BREAK_SYSTEM_PACKAGES: "1" - PYTHONUNBUFFERED: "1" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_OPT_USE_TOPK_V2: "1" - SGLANG_TIMEOUT_KEEP_ALIVE: "900" - - decode_environment: + prefill_environment: &worker_environment HF_HOME: "/hf_hub_cache" HF_HUB_CACHE: "/hf_hub_cache" PIP_BREAK_SYSTEM_PACKAGES: "1" @@ -76,6 +54,7 @@ backend: SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" SGLANG_OPT_USE_TOPK_V2: "1" SGLANG_TIMEOUT_KEEP_ALIVE: "900" + decode_environment: *worker_environment sglang_config: prefill: @@ -93,7 +72,7 @@ backend: enable-dsa-cache-layer-split: true disaggregation-transfer-backend: mooncake disaggregation-mode: prefill - kv-cache-dtype: fp8_e4m3 + kv-cache-dtype: bfloat16 context-length: 1048576 max-total-tokens: 1048576 chunked-prefill-size: 32768 @@ -102,11 +81,6 @@ backend: watchdog-timeout: 86400 stream-interval: 60 enable-metrics: true - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-ratio: 1 - hicache-io-backend: direct - hicache-mem-layout: layer_first decode: host: 0.0.0.0 @@ -115,17 +89,22 @@ backend: trust-remote-code: true tool-call-parser: glm47 reasoning-parser: glm45 - tp-size: 16 + tp-size: 8 + dp-size: 8 ep-size: 1 + enable-dp-attention: true disaggregation-transfer-backend: mooncake disaggregation-mode: decode - disaggregation-decode-enable-radix-cache: true - kv-cache-dtype: fp8_e4m3 + kv-cache-dtype: bfloat16 + dsa-decode-backend: flashmla_sparse context-length: 1048576 max-total-tokens: 1048576 mem-fraction-static: 0.85 - max-running-requests: 32 - cuda-graph-max-bs: 32 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + enable-hisparse: true + hisparse-config: '{"top_k":2048,"device_buffer_size":4096,"host_to_device_ratio":5}' watchdog-timeout: 86400 stream-interval: 60 enable-metrics: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index ba20c2bd03..0f725f92e3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7829,13 +7829,9 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true -# GLM-5.2 FP8 full-context AgentX qualification on H200. SGLang prefill CP -# runs inside one TP8 worker, so the matrix describes the effective attention -# topology as TP1 x PCP8 while the recipe launches eight ranks. LayerSplit -# shards DSA KV/indexer cache layers across those ranks. Decode remains TP16: -# upstream DSA decode-context parallelism is not merged, and two H200 nodes -# are required to leave runtime headroom beside the explicit 1M FP8 KV pool. -glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: +# GLM-5.2 FP8 full-context AgentX on H200. PCP8 prefill processes +# the 1M context while TP8/DPA decode uses HiSparse-backed host DRAM. +glm5.2-fp8-h200-dynamo-sglang-agentic-hisparse-pareto: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 model: zai-org/GLM-5.2-FP8 model-prefix: glm5.2 @@ -7851,96 +7847,9 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: - dram-utilization: 0.80 search-space: - spec-decoding: none - # Isolate each point: long-prefix restore failures at one concurrency - # must not discard completed points or prevent later points from running. - conc-list: [1] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [2] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [3] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [4] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [5] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [6] + conc-list: [1, 2, 3, 4, 5, 6, 7, 8] kv-offloading: dram - kv-offload-backend: { name: hicache } + kv-offload-backend: { name: hisparse } prefill: num-worker: 1 tp: 1 @@ -7948,29 +7857,12 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-pcp-hicache: ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml" decode: num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [8] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 1 - pcp-size: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp16-hicache.yaml" - decode: - num-worker: 1 - tp: 16 + tp: 8 ep: 1 - dp-attn: false + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 38efdbf219..f98d85d838 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5011,3 +5011,13 @@ - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms plus a new TP8 GPU-resident arm -- TP8 GPU-resident [1,2,4,6,8], TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258 +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-hisparse-pareto + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 FP8 full-context H200 AgentX with one PCP8 prefill worker and one TP8/DPA decode worker across two H200 nodes" + - "Use SGLang HiSparse to retain the full KV cache in pinned host DRAM while a sparse decode working set remains in HBM; this replaces the TP16 HiCache design that could not restore literal 1M-token prefixes reliably" + - "Run the complete concurrency curve [1, 2, 3, 4, 5, 6, 7, 8]; workflow run 29820102138 completed all eight points with zero benchmark errors, peaking at 312.63 total tok/s/GPU at concurrency 4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2275 +