From 6a59ae81c21d65fa1c14cc5f8d3ff880002c619a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 12:53:04 -0500 Subject: [PATCH 1/9] feat: add MiniMax-M2.7 EAGLE B200 experiment [skip-sweep] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pinned public NVFP4 target and full-vocabulary EAGLE3 draft staging, TP4/TP8 8K/1K and AgentX matrices, and shared SGLang launch logic with chat templates.\n\n中文：新增固定版本的公开 NVFP4 目标模型与全词表 EAGLE3 草稿模型暂存逻辑、TP4/TP8 的 8K/1K 与 AgentX 扫描矩阵，以及启用聊天模板的共享 SGLang 启动逻辑。 --- .../minimaxm2.7_fp4_b200_sglang_mtp.sh | 5 + .../minimaxm2.7_fp4_b200_sglang_mtp.sh | 210 ++++++++++++++++++ configs/nvidia-master.yaml | 25 +++ perf-changelog.yaml | 8 + runners/launch_b200-dgxc.sh | 13 +- 5 files changed, 260 insertions(+), 1 deletion(-) create mode 100755 benchmarks/single_node/agentic/minimaxm2.7_fp4_b200_sglang_mtp.sh create mode 100755 benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/minimaxm2.7_fp4_b200_sglang_mtp.sh new file mode 100755 index 0000000000..b33b18f7df --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export SCENARIO_TYPE=agentic-coding +source "$(dirname "${BASH_SOURCE[0]}")/../fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh" diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh new file mode 100755 index 0000000000..dd2265b2d7 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# MiniMax-M2.7 NVFP4 on B200 with the public full-vocabulary EAGLE3 draft. +# This entrypoint serves both fixed-sequence and AgentX wrappers so the target, +# draft, speculative settings, and chat behavior cannot drift between them. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + EP_SIZE \ + CONC \ + PORT \ + PRECISION \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME + +if [[ "$PRECISION" != "fp4" || "$EP_SIZE" != "1" ]]; then + echo "MiniMax-M2.7 EAGLE3 expects NVFP4 with pure tensor parallelism, got precision=$PRECISION EP=$EP_SIZE" >&2 + exit 1 +fi +if [[ "$TP" != "4" && "$TP" != "8" ]]; then + echo "MiniMax-M2.7 EAGLE3 supports the configured TP4/TP8 search space, got TP=$TP" >&2 + exit 1 +fi + +readonly TARGET_MODEL="nvidia/MiniMax-M2.7-NVFP4" +readonly TARGET_REVISION="e79701cb1f9dce8fe5395b9ed2b20170beebecde" +readonly DRAFT_MODEL="asherszhang/MiniMax-M2.7-EAGLE3-draft-vocab200k" +readonly DRAFT_REVISION="252b54f7d05a5d0db7734563ae10e2e6a81d6a7d" + +if [[ "$MODEL" != "$TARGET_MODEL" && "$MODEL" != /* ]]; then + echo "Unexpected target model: $MODEL (expected $TARGET_MODEL)" >&2 + exit 1 +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + TARGET_MODEL_PATH="$MODEL_PATH" +elif [[ "$MODEL" == /* ]]; then + TARGET_MODEL_PATH="$MODEL" +else + TARGET_MODEL_PATH="$TARGET_MODEL" +fi + +if [[ "$TARGET_MODEL_PATH" == /* ]]; then + MODEL_ROOT=$(dirname "$TARGET_MODEL_PATH") + DRAFT_MODEL_PATH="${DRAFT_MODEL_PATH:-$MODEL_ROOT/MiniMax-M2.7-EAGLE3-draft-vocab200k}" + mkdir -p "$TARGET_MODEL_PATH" "$DRAFT_MODEL_PATH" + exec 8>"$MODEL_ROOT/.minimax-m2.7-eagle3-stage.lock" + flock -w 7200 8 + if [[ ! -f "$TARGET_MODEL_PATH/config.json" || \ + ! -f "$TARGET_MODEL_PATH/model.safetensors.index.json" || \ + $(find "$TARGET_MODEL_PATH" -maxdepth 1 -name 'model-*-of-00015.safetensors' | wc -l) -ne 15 ]]; then + hf download "$TARGET_MODEL" --revision "$TARGET_REVISION" --local-dir "$TARGET_MODEL_PATH" + fi + if [[ ! -f "$DRAFT_MODEL_PATH/config.json" || ! -f "$DRAFT_MODEL_PATH/model.safetensors" ]]; then + hf download "$DRAFT_MODEL" --revision "$DRAFT_REVISION" --local-dir "$DRAFT_MODEL_PATH" + fi + flock -u 8 +else + hf download "$TARGET_MODEL" --revision "$TARGET_REVISION" + hf download "$DRAFT_MODEL" --revision "$DRAFT_REVISION" + DRAFT_MODEL_PATH="$DRAFT_MODEL" +fi + +python3 - "$TARGET_MODEL_PATH" "$DRAFT_MODEL_PATH" <<'PY' +import json +import sys +from pathlib import Path + +target_path, draft_path = map(Path, sys.argv[1:]) +target = json.loads((target_path / "config.json").read_text()) +draft = json.loads((draft_path / "config.json").read_text()) +quant = target.get("quantization_config", {}) +if quant.get("quant_algo") != "NVFP4" or quant.get("quant_method") != "modelopt": + raise SystemExit(f"Unexpected NVFP4 target configuration: {quant}") +if target.get("vocab_size") != 200064: + raise SystemExit(f"Unexpected target vocabulary: {target.get('vocab_size')}") +if draft.get("architectures") != ["LlamaForCausalLMEagle3"]: + raise SystemExit(f"Unexpected EAGLE3 architecture: {draft.get('architectures')}") +if draft.get("vocab_size") != 200064 or draft.get("num_hidden_layers") != 1: + raise SystemExit( + f"Unexpected EAGLE3 draft contract: vocab={draft.get('vocab_size')} " + f"layers={draft.get('num_hidden_layers')}" + ) +print(f"Validated target={target_path} draft={draft_path}") +PY + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" +fi +nvidia-smi + +IS_AGENTIC=false +if [[ "${SCENARIO_TYPE:-fixed-seq-len}" == "agentic-coding" ]]; then + IS_AGENTIC=true + check_env_vars DURATION RESULT_DIR KV_OFFLOADING TOTAL_CPU_DRAM_GB + if [[ "$KV_OFFLOADING" != "none" ]]; then + echo "This low-concurrency EAGLE3 experiment requires GPU-resident KV cache" >&2 + exit 1 + fi + export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}" + resolve_trace_source + install_agentic_deps + OUTPUT_DIR="$RESULT_DIR" + CONTEXT_LENGTH=196608 + MAX_RUNNING_REQUESTS=$((CONC * 2)) + (( MAX_RUNNING_REQUESTS < 8 )) && MAX_RUNNING_REQUESTS=8 +else + check_env_vars ISL OSL MAX_MODEL_LEN + OUTPUT_DIR="$PWD" + CONTEXT_LENGTH="$MAX_MODEL_LEN" + MAX_RUNNING_REQUESTS=$((CONC * 2)) +fi +mkdir -p "$OUTPUT_DIR" + +SERVER_LOG="$OUTPUT_DIR/server.log" +SERVER_PID="" +GPU_MONITOR_STARTED=false + +cleanup() { + local exit_code=$? + trap - EXIT INT TERM + set +e + if [[ "$GPU_MONITOR_STARTED" == "true" ]]; then + stop_gpu_monitor + fi + stop_background_process_tree "$SERVER_PID" "MiniMax-M2.7 EAGLE3 server" 60 + exit "$exit_code" +} +trap cleanup EXIT INT TERM + +start_gpu_monitor --output "$OUTPUT_DIR/gpu_metrics.csv" +GPU_MONITOR_STARTED=true + +SGLANG_CMD=( + python3 -m sglang.launch_server + --model-path "$TARGET_MODEL_PATH" + --served-model-name "$TARGET_MODEL" + --host 0.0.0.0 + --port "$PORT" + --trust-remote-code + --tp "$TP" + --ep-size "$EP_SIZE" + --quantization modelopt_fp4 + --dtype bfloat16 + --attention-backend triton + --moe-runner-backend flashinfer_cutlass + --speculative-algorithm EAGLE3 + --speculative-draft-model-path "$DRAFT_MODEL_PATH" + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 + --speculative-draft-model-quantization unquant + --speculative-draft-attention-backend triton + --reasoning-parser minimax + --tool-call-parser minimax-m2 + --context-length "$CONTEXT_LENGTH" + --chunked-prefill-size 16384 + --max-running-requests "$MAX_RUNNING_REQUESTS" + --mem-fraction-static 0.90 + --enable-metrics + --watchdog-timeout 1800 +) + +printf '%q ' "${SGLANG_CMD[@]}" | tee "$OUTPUT_DIR/sglang_command.txt" +printf '\n' | tee -a "$OUTPUT_DIR/sglang_command.txt" +"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +mtp_metrics=$(curl -fsS "http://127.0.0.1:$PORT/metrics") +if ! grep -m1 '^sglang:spec_accept_length' <<< "$mtp_metrics"; then + echo "EAGLE3 acceptance metrics are absent; refusing to publish a non-speculative result" >&2 + exit 1 +fi + +if [[ "$IS_AGENTIC" == "true" ]]; then + # AgentX sends structured chat messages to /v1/chat/completions, so the + # target checkpoint's native chat template is applied by the server. + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +else + # EAGLE3 was trained on chat-formatted prompts; keep the benchmark client + # on the target checkpoint's native chat template. + run_benchmark_serving \ + --model "$TARGET_MODEL_PATH" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir "$PWD" \ + --trust-remote-code \ + --use-chat-template + + if [[ "${RUN_EVAL:-false}" == "true" ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary + fi +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3aac1b8378..05df85b23d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7875,3 +7875,28 @@ glm5.2-fp4-b300-sglang-agentic: search-space: - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } } + +# Experimental public-weight MiniMax-M2.7 NVFP4 + EAGLE3 B200 baseline. The +# target and full-vocabulary draft revisions are pinned in the benchmark +# entrypoint; both scenarios use the same upstream SGLang runtime and 3/1/4 +# speculative chain. +minimaxm2.7-fp4-b200-sglang-eagle: + image: lmsysorg/sglang:v0.5.15.post1-cu130 + model: nvidia/MiniMax-M2.7-NVFP4 + model-prefix: minimaxm2.7 + runner: cluster:b200-dgxc + precision: fp4 + framework: sglang + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 4, ep: 1, conc-list: [1, 2, 4, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, conc-list: [1, 2, 4, 8], spec-decoding: mtp } + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 892e3ee2b6..f3489e2833 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4974,3 +4974,11 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - minimaxm2.7-fp4-b200-sglang-eagle + description: + - "Experiment with MiniMax-M2.7 NVFP4 and a public full-vocabulary EAGLE3 draft on B200 using upstream SGLang v0.5.15.post1." + - "Run 8K/1K and AgentX at pure TP4/TP8 with chat templates and a 3-step, top-k-1, 4-draft-token speculative chain." + - "Pin nvidia/MiniMax-M2.7-NVFP4 and asherszhang/MiniMax-M2.7-EAGLE3-draft-vocab200k revisions in the benchmark preflight." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 58051e6405..9d68e71adb 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -60,6 +60,11 @@ elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-nvfp4" +elif [[ $MODEL_PREFIX == "minimaxm2.7" && $PRECISION == "fp4" ]]; then + # Public NVFP4 target and EAGLE3 draft are staged in the runner-writable + # shared model tree by the benchmark's revision-pinned preflight. + export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M2.7-NVFP4" + export SRT_SLURM_MODEL_PREFIX="minimax-m2.7-nvfp4" elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" export SRT_SLURM_MODEL_PREFIX="gptoss" @@ -397,6 +402,12 @@ else # pulling from the HF hub cache. Bench scripts skip `hf download` when # MODEL is a local path. export MODEL="$MODEL_PATH" + MODEL_MOUNT_SPEC="$MODEL_PATH:$MODEL_PATH" + if [[ "$MODEL_PREFIX" == "minimaxm2.7" ]]; then + MODEL_ROOT=$(dirname "$MODEL_PATH") + mkdir -p "$MODEL_PATH" "$MODEL_ROOT/MiniMax-M2.7-EAGLE3-draft-vocab200k" + MODEL_MOUNT_SPEC="$MODEL_ROOT:$MODEL_ROOT" + fi FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') # Prefer a framework-tagged script (e.g. dsv4_fp4_b200_vllm.sh) so models @@ -447,7 +458,7 @@ else srun --jobid=$JOB_ID \ --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_PATH:$MODEL_PATH,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ + --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$MODEL_MOUNT_SPEC,$AIPERF_MMAP_CACHE_HOST_PATH:/aiperf_mmap_cache \ --no-container-mount-home \ --container-workdir=$CONTAINER_MOUNT_DIR \ --no-container-entrypoint --export=ALL,PORT=8888,AIPERF_DATASET_MMAP_CACHE_DIR=/aiperf_mmap_cache \ From ade27522b374a1b7cbf2f53bb9b3e515af43914a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 12:53:30 -0500 Subject: [PATCH 2/9] chore: link MiniMax-M2.7 experiment PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the draft PR URL in the append-only performance changelog before starting benchmark jobs.\n\n中文：在启动基准测试任务前，将草稿 PR 链接记录到仅追加的性能变更日志中。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f3489e2833..ac7857064d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4981,4 +4981,4 @@ - "Experiment with MiniMax-M2.7 NVFP4 and a public full-vocabulary EAGLE3 draft on B200 using upstream SGLang v0.5.15.post1." - "Run 8K/1K and AgentX at pure TP4/TP8 with chat templates and a 3-step, top-k-1, 4-draft-token speculative chain." - "Pin nvidia/MiniMax-M2.7-NVFP4 and asherszhang/MiniMax-M2.7-EAGLE3-draft-vocab200k revisions in the benchmark preflight." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2289 From 931d9e86524227a7948842e55fc6ef3863b32846 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 13:07:28 -0500 Subject: [PATCH 3/9] perf: prefetch MiniMax checkpoints from Lustre MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partition checkpoint prefetching across tensor-parallel ranks to avoid redundant network reads during repeated B200 engine startups.\n\n中文：在张量并行 rank 之间划分检查点预取，避免 B200 推理引擎重复启动时产生冗余网络读取。 --- .../single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh index dd2265b2d7..dd6ed49ab2 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -161,6 +161,7 @@ SGLANG_CMD=( --tool-call-parser minimax-m2 --context-length "$CONTEXT_LENGTH" --chunked-prefill-size 16384 + --weight-loader-prefetch-checkpoints --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static 0.90 --enable-metrics From 1401c61d23418be12ffd6317b7fec0cf7b7befdf Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 13:17:37 -0500 Subject: [PATCH 4/9] fix: leave B200 headroom for EAGLE graphs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the SGLang static memory fraction from 0.90 to 0.85 after TP4 and TP8 smoke runs OOMed while capturing the 16K prefill CUDA graph.\n\n中文：TP4 和 TP8 冒烟测试在捕获 16K 预填充 CUDA Graph 时显存溢出，因此将 SGLang 静态显存比例从 0.90 降至 0.85，为 EAGLE 图捕获预留空间。 --- .../fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh index dd6ed49ab2..bc5c10350f 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -163,7 +163,7 @@ SGLANG_CMD=( --chunked-prefill-size 16384 --weight-loader-prefetch-checkpoints --max-running-requests "$MAX_RUNNING_REQUESTS" - --mem-fraction-static 0.90 + --mem-fraction-static 0.85 --enable-metrics --watchdog-timeout 1800 ) From 6110522c86488bdf7f3f464c11a9f610f4ee55d7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 13:22:52 -0500 Subject: [PATCH 5/9] ci: route MiniMax experiment to idle B200 partition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select gpu-1 only for the MiniMax-M2.7 experiment after confirming all ten nodes are healthy and idle while gpu-2 is saturated. Other B200 benchmark families keep their existing partition.\n\n中文：确认 gpu-1 的十个节点均健康且空闲、gpu-2 已满载后，仅将 MiniMax-M2.7 实验调度到 gpu-1；其他 B200 基准测试仍使用原分区。 --- runners/launch_b200-dgxc.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 9d68e71adb..a547845528 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -65,6 +65,9 @@ elif [[ $MODEL_PREFIX == "minimaxm2.7" && $PRECISION == "fp4" ]]; then # shared model tree by the benchmark's revision-pinned preflight. export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M2.7-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.7-nvfp4" + # Keep this experiment off the saturated gpu-2 partition. All ten gpu-1 + # B200 nodes are currently healthy and idle. + SLURM_PARTITION="gpu-1" elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" export SRT_SLURM_MODEL_PREFIX="gptoss" @@ -431,10 +434,9 @@ else CONTAINER_MOUNT_DIR=/workspace fi - # b200-dgxc cluster was re-partitioned to gpu-1 / gpu-2; the prior gpu-10 - # and gpu-15 names no longer exist. gpu-2 currently has 10 fully-idle GPU - # nodes (all of gpu-2-[0-9]); gpu-1 has 2 drained (gpu-1-4, gpu-1-8). We - # land on gpu-2 to avoid drained nodes and skip the per-node excludes. + # b200-dgxc is partitioned into gpu-1 / gpu-2; the prior gpu-10 and gpu-15 + # names no longer exist. Model-specific mappings above may select the + # currently idle partition without affecting other benchmark families. export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" From 9bbb9d95524a31d6ba6e9c3f845b42d7ccaa45e1 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 13:26:55 -0500 Subject: [PATCH 6/9] fix: retain authorized B200 Slurm partition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep benchmark jobs on gpu-2 after Slurm association inspection confirmed sa-shared has only gpu-2_qos; gpu-1 rejects submissions despite idle nodes.\n\n中文：检查 Slurm 关联后确认 sa-shared 仅拥有 gpu-2_qos，因此继续在 gpu-2 上运行基准测试；gpu-1 即使有空闲节点也会拒绝提交。 --- runners/launch_b200-dgxc.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index a547845528..7cc98ca391 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -65,9 +65,6 @@ elif [[ $MODEL_PREFIX == "minimaxm2.7" && $PRECISION == "fp4" ]]; then # shared model tree by the benchmark's revision-pinned preflight. export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M2.7-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.7-nvfp4" - # Keep this experiment off the saturated gpu-2 partition. All ten gpu-1 - # B200 nodes are currently healthy and idle. - SLURM_PARTITION="gpu-1" elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" export SRT_SLURM_MODEL_PREFIX="gptoss" @@ -435,8 +432,8 @@ else fi # b200-dgxc is partitioned into gpu-1 / gpu-2; the prior gpu-10 and gpu-15 - # names no longer exist. Model-specific mappings above may select the - # currently idle partition without affecting other benchmark families. + # names no longer exist. sa-shared's benchmark association currently has + # only gpu-2_qos, so idle gpu-1 nodes cannot accept these jobs. export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" From e8e80c851883a37f477aaa2124231b1cd309e277 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 13:34:38 -0500 Subject: [PATCH 7/9] ci: allow MiniMax jobs to backfill B200 nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a 150-minute model-scoped Slurm limit instead of the launcher-wide eight-hour default, retaining setup, warmup, and one-hour AgentX headroom while permitting backfill.\n\n中文：为该模型使用 150 分钟的 Slurm 时限，替代启动器默认的八小时；既覆盖准备、预热和一小时 AgentX 测量，也允许调度器进行回填。 --- runners/launch_b200-dgxc.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 7cc98ca391..9e4c2ef088 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -65,6 +65,9 @@ elif [[ $MODEL_PREFIX == "minimaxm2.7" && $PRECISION == "fp4" ]]; then # shared model tree by the benchmark's revision-pinned preflight. export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M2.7-NVFP4" export SRT_SLURM_MODEL_PREFIX="minimax-m2.7-nvfp4" + # Covers the one-hour AgentX measurement plus setup and warmup while still + # allowing Slurm to backfill this experiment ahead of large reservations. + SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-150}" elif [[ $MODEL_PREFIX == "gptoss" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/gpt-oss-120b" export SRT_SLURM_MODEL_PREFIX="gptoss" From ed243584ec303cf34f56d22741a30a5ae63e5593 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 15:15:33 -0500 Subject: [PATCH 8/9] fix: cap MiniMax AgentX replay context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bind the AgentX replay limit to the 196608-token SGLang server window so oversized Weka trajectory turns are excluded before they become deterministic HTTP 400 failures. 中文：将 AgentX 重放上限绑定到 SGLang 的 196608 token 上下文窗口，使超长 Weka 轨迹轮次在发送前被排除，避免确定性的 HTTP 400 失败。 --- .../single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh index bc5c10350f..25cfccf83c 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -108,6 +108,7 @@ if [[ "${SCENARIO_TYPE:-fixed-seq-len}" == "agentic-coding" ]]; then install_agentic_deps OUTPUT_DIR="$RESULT_DIR" CONTEXT_LENGTH=196608 + export MAX_MODEL_LEN="$CONTEXT_LENGTH" MAX_RUNNING_REQUESTS=$((CONC * 2)) (( MAX_RUNNING_REQUESTS < 8 )) && MAX_RUNNING_REQUESTS=8 else From 4c43c23d0aaa8fabc383a917f1cc781834bde4aa Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 21 Jul 2026 11:55:14 -0500 Subject: [PATCH 9/9] fix: drain slow MiniMax AgentX responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow up to 30 minutes for AgentX requests admitted during the measurement window to finish. TP4 was healthy and decoding when the 30-second AIPerf default grace cancelled its only smoke profile request. 中文：允许在测量窗口内已接收的 AgentX 请求最多用 30 分钟完成。TP4 服务保持健康并持续解码，但 AIPerf 默认的 30 秒宽限期取消了冒烟测试中唯一的测量请求。 --- .../fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh index 25cfccf83c..e76d0ac29e 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm2.7_fp4_b200_sglang_mtp.sh @@ -186,6 +186,12 @@ if [[ "$IS_AGENTIC" == "true" ]]; then # AgentX sends structured chat messages to /v1/chat/completions, so the # target checkpoint's native chat template is applied by the server. build_replay_cmd "$RESULT_DIR" + # TP4 can take several minutes to finish a long response that was already + # admitted near the end of the measurement window. The AIPerf default is + # only 30 seconds, which cancelled the sole profiling request in the TP4 + # smoke while the healthy server was still decoding it. Bound the drain at + # 30 minutes without extending the measured request-admission window. + REPLAY_CMD+=" --benchmark-grace-period 1800" REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" run_agentic_replay_and_write_outputs "$RESULT_DIR" else