diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml new file mode 100644 index 0000000000..97c5d0ceeb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml @@ -0,0 +1,130 @@ +name: "disagg-h200-1p1d-pcp8-tp8-dp8-hisparse" + +# Experimental HiSparse decode. This is not HiCache: full KV lives in pinned +# host memory while a fixed sparse working set remains in HBM. + +model: + path: "hf:zai-org/GLM-5.2-FP8" + container: "lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8" + precision: "fp8" + +dynamo: + install: true + wheel: "1.3.0.dev1" + +slurm: + time_limit: "8:00:00" + +resources: + gpu_type: h200 + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 32 + +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Correlation-ID + env: + DYN_ROUTER_TEMPERATURE: "10000000" + PIP_BREAK_SYSTEM_PACKAGES: "1" + args: + router-mode: "kv" + router-reset-states: true + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +backend: + type: sglang + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache" + PIP_BREAK_SYSTEM_PACKAGES: "1" + PYTHONUNBUFFERED: "1" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_OPT_USE_TOPK_V2: "1" + SGLANG_TIMEOUT_KEEP_ALIVE: "900" + decode_environment: *worker_environment + + sglang_config: + prefill: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + ep-size: 1 + attn-cp-size: 8 + enable-prefill-cp: true + cp-strategy: interleave + enable-dsa-cache-layer-split: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: prefill + kv-cache-dtype: bfloat16 + context-length: 1048576 + max-total-tokens: 1048576 + chunked-prefill-size: 32768 + mem-fraction-static: 0.85 + max-running-requests: 32 + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + + decode: + host: 0.0.0.0 + model-path: /model/ + served-model-name: zai-org/GLM-5.2-FP8 + trust-remote-code: true + tool-call-parser: glm47 + reasoning-parser: glm45 + tp-size: 8 + dp-size: 8 + ep-size: 1 + enable-dp-attention: true + disaggregation-transfer-backend: mooncake + disaggregation-mode: decode + kv-cache-dtype: bfloat16 + dsa-decode-backend: flashmla_sparse + context-length: 1048576 + max-total-tokens: 1048576 + mem-fraction-static: 0.85 + max-running-requests: 200 + page-size: 64 + disable-radix-cache: true + enable-hisparse: true + hisparse-config: '{"top_k":2048,"device_buffer_size":4096,"host_to_device_ratio":5}' + watchdog-timeout: 86400 + stream-interval: 60 + enable-metrics: true + +sbatch_directives: + mem: "0" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index dd9e40ec87..0f725f92e3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7828,6 +7828,41 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 16 ep: 16 dp-attn: true + +# GLM-5.2 FP8 full-context AgentX on H200. PCP8 prefill processes +# the 1M context while TP8/DPA decode uses HiSparse-backed host DRAM. +glm5.2-fp8-h200-dynamo-sglang-agentic-hisparse-pareto: + image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + model: zai-org/GLM-5.2-FP8 + model-prefix: glm5.2 + runner: cluster:h200-dgxc + precision: fp8 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.3.0.dev1" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: none + conc-list: [1, 2, 3, 4, 5, 6, 7, 8] + kv-offloading: dram + kv-offload-backend: { name: hisparse } + prefill: + num-worker: 1 + tp: 1 + pcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5.2/agentic/disagg-h200-1p1d-pcp8-tp8-dp8-hisparse.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 38efdbf219..f98d85d838 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5011,3 +5011,13 @@ - "dsv4-fp4-b300-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms plus a new TP8 GPU-resident arm -- TP8 GPU-resident [1,2,4,6,8], TP4 GPU-resident [1,2,4,6,8,12,16,20], TP4 SimpleCPU [20,24,28,32,36,40], DEP4 [32,40,48,56], DEP8 [64,96,112,128,144,160,176,192,224]; routed via spec-decoding=mtp to dsv4_fp4_b300_vllm_mtp.sh (FULL_DECODE_ONLY cudagraph capture sizes in TOKENS = num_seqs*4)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2258 +- config-keys: + - glm5.2-fp8-h200-dynamo-sglang-agentic-hisparse-pareto + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 FP8 full-context H200 AgentX with one PCP8 prefill worker and one TP8/DPA decode worker across two H200 nodes" + - "Use SGLang HiSparse to retain the full KV cache in pinned host DRAM while a sparse decode working set remains in HBM; this replaces the TP16 HiCache design that could not restore literal 1M-token prefixes reliably" + - "Run the complete concurrency curve [1, 2, 3, 4, 5, 6, 7, 8]; workflow run 29820102138 completed all eight points with zero benchmark errors, peaking at 312.63 total tok/s/GPU at concurrency 4" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2275 + diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 00c6cc4977..c23fabb947 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -9,6 +9,17 @@ set -x if [[ "$IS_MULTINODE" == "true" ]]; then + # Resolve InferenceX-owned recipes before entering the cloned srt-slurm + # checkout. AgentX also needs persistent host caches on every worker. + CONFIG_PATH="${CONFIG_FILE%%:*}" + LOCAL_CONFIG_FILE="" + LOCAL_CONFIG_PATH="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" + if [[ -n "${CONFIG_FILE:-}" && -f "$LOCAL_CONFIG_PATH" ]]; then + LOCAL_CONFIG_FILE="$LOCAL_CONFIG_PATH" + fi + AIPERF_MMAP_CACHE_HOST_PATH="/home/sa-shared/gharunners/ai-perf-cache" + HF_HUB_CACHE_HOST_PATH="/models/gharunners/hf-hub-cache" + # MODEL_PATH: Override with pre-downloaded paths on H200 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H200 cluster. @@ -16,6 +27,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/models/DeepSeek-R1-0528" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" + elif [[ $MODEL_PREFIX == "glm5.2" && $PRECISION == "fp8" ]]; then + export MODEL_PATH="${GLM52_FP8_MODEL_PATH:-/models/GLM-5.2-FP8}" + if [[ ! -d "$MODEL_PATH" ]]; then + export MODEL_PATH="hf:zai-org/GLM-5.2-FP8" + fi + export SRT_SLURM_MODEL_PREFIX="glm5.2-fp8" else echo "Unsupported model prefix/precision for dynamo-sglang: $MODEL_PREFIX/$PRECISION" exit 1 @@ -42,7 +59,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then fi # TODO(CJQ): make first class upon srt-slurm upstream refactor - if [[ "$IS_AGENTIC" == "1" ]]; then + if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.2" ]]; then + # v1.0.10 contains the custom benchmark hook, session-affinity + # frontend, and large-request nginx fix used by SGLang AgentX recipes. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.10 + elif [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" else @@ -71,7 +94,11 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ $FRAMEWORK == "dynamo-sglang" ]]; then # SGLang container mapping - SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + if [[ $MODEL_PREFIX == "glm5.2" ]]; then + SQUASH_FILE="/data/gharunners/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + else + SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + fi CONTAINER_KEY="$IMAGE" elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) @@ -79,6 +106,24 @@ if [[ "$IS_MULTINODE" == "true" ]]; then SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's|nvcr.io/||' | sed 's/[\/:@#]/+/g').sqsh" fi + if ! unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + DOCKER_IMAGE=$(echo "$IMAGE" | sed 's/#/\//g') + LOCK_FILE="${SQUASH_FILE}.lock" + srun --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" \ + --nodes=1 --ntasks=1 --time=30 --job-name="$RUNNER_NAME" \ + bash -c " + exec 9>\"$LOCK_FILE\" + flock -w 1800 9 || { echo 'Failed to acquire $LOCK_FILE'; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" >/dev/null 2>&1; then + exit 0 + fi + rm -f \"$SQUASH_FILE\" + export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot + mkdir -p \"\$ENROOT_CACHE_PATH\" + enroot import -o \"$SQUASH_FILE\" docker://$DOCKER_IMAGE + " + fi + export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" @@ -98,6 +143,10 @@ gpus_per_node: 8 network_interface: "" # Path to srtctl repo root (where the configs live) srtctl_root: "${SRTCTL_ROOT}" +# Persistent AgentX dataset caches mounted into every worker container. +default_mounts: + "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" + "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" # Model path aliases model_paths: "${SRT_SLURM_MODEL_PREFIX}": "${MODEL_PATH}" @@ -120,6 +169,13 @@ EOF echo "Running make setup..." make setup ARCH=x86_64 + # make setup refreshes the srt-slurm recipe tree. Overlay the checked-in + # InferenceX recipe afterwards so it remains available to srtctl apply. + if [[ -n "$LOCAL_CONFIG_FILE" ]]; then + mkdir -p "$(dirname "$CONFIG_PATH")" + cp "$LOCAL_CONFIG_FILE" "$CONFIG_PATH" + fi + # Export eval-related env vars for srt-slurm post-benchmark eval export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" @@ -132,10 +188,14 @@ EOF fi # Override the job name in the config file with the runner name - sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "${CONFIG_FILE%%:*}" - printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "${CONFIG_FILE%%:*}" - SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) + sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH" + printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH" + WORKLOAD_TAG="${ISL}x${OSL}" + if [[ "$IS_AGENTIC" == "1" ]]; then + WORKLOAD_TAG="agentic" + fi + SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h200,${MODEL_PREFIX},${PRECISION},${WORKLOAD_TAG},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT" # Extract JOB_ID from srtctl output