fix(evals): stabilize DSV4 quality sweep

Author: adibarra

benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh

@@ -110,8 +110,12 @@ MAX_RUNNING_REQUESTS=$((2 * CONC))
 CUDA_GRAPH_MAX_BS=$CONC
 [ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128
 
-export SGLANG_DEFAULT_THINKING=1
-export SGLANG_DSV4_REASONING_EFFORT=high
+if [ "${IS_AGENTIC:-0}" = "1" ]; then
+    export SGLANG_DEFAULT_THINKING=1
+    export SGLANG_DSV4_REASONING_EFFORT=high
+else
+    unset SGLANG_DEFAULT_THINKING SGLANG_DSV4_REASONING_EFFORT
+fi
 export SGLANG_USE_ROCM700A=0
 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
 export AITER_BF16_FP8_MOE_BOUND=0

configs/amd-master.yaml

@@ -1984,7 +1984,7 @@ dsv4-fp4-mi355x-sglang-mtp:
 # gpu-mem-util=0.6. TP8 sweeps conc 4-64; DEP8 has a single conc=64
 # probe to validate the ROCm DP+EP path.
 dsv4-fp4-mi355x-vllm:
-  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  image: vllm/vllm-openai-rocm:nightly-67fe73b2b47e2927af1e5ec6fb91e16e3b075940
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: mi355x
@@ -2008,10 +2008,10 @@ dsv4-fp4-mi355x-vllm:
 # vllm-project/vllm#43385 (ROCm DeepSeek-V4 MTP, merged 2026-05-24, included in
 # v0.22.0). Full conc 4-512 range maps the complete crossover curve: MTP wins
 # at low batch (PR perf data: +75% @ conc1, +38% @ conc8) and falls behind STP
-# above ~conc32 (-37% @ conc32). Image reuses the base entry's v0.22.0 ROCm
-# build, which already contains the MTP commit.
+# above ~conc32 (-37% @ conc32). The pinned nightly contains both the base
+# DeepSeek-V4 ROCm path and the MTP implementation.
 dsv4-fp4-mi355x-vllm-mtp:
-  image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa
+  image: vllm/vllm-openai-rocm:nightly-67fe73b2b47e2927af1e5ec6fb91e16e3b075940
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: mi355x

utils/evals/EVALS.md

@@ -156,7 +156,7 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]'
 | `EVAL_LIMIT` | empty | Limit eval to first N instances (smoke tests); empty = full set |
 
 ### Score validation
-`utils/evals/validate_scores.py` checks full-split eval results against thresholds in `utils/evals/thresholds.yaml`. First-N smoke slices skip the quality gate because their scores are not statistically representative. Validation runs after artifact upload so full-run results are preserved even if the gate fails.
+`utils/evals/validate_scores.py` checks full-split eval results against thresholds in `utils/evals/thresholds.json`. The JSON format keeps validation independent of optional PyYAML installations on runner hosts. First-N smoke slices skip the quality gate because their scores are not statistically representative. Validation runs after artifact upload so full-run results are preserved even if the gate fails.
 
 ### Adding a new eval task
 
@@ -214,7 +214,7 @@ append_lm_eval_summary
   is derived from the YAML's `dataset_path` so generation and scoring can't diverge;
   `SWEBENCH_DATASET`, if set, must match it (mismatch fails fast).
 - Scoring runs on Modal remote sandboxes in CI (`SWEBENCH_USE_MODAL=true`, no Docker on the GPU
-  nodes); local Docker scoring needs ~120 GB disk. The `thresholds.yaml` gate is `0.50`, calibrated
+  nodes); local Docker scoring needs ~120 GB disk. The `thresholds.json` gate is `0.50`, calibrated
   from full-split runs (54%); historical 50-instance slices scored 62–76%.
 
 ## Task files

utils/evals/thresholds.json

@@ -0,0 +1,36 @@
+{
+  "default": {
+    "aime26": 0.3,
+    "gsm8k": 0.9,
+    "gpqa_diamond_cot_n_shot": 0.3,
+    "swebench_lite": 0.5
+  },
+  "models": {
+    "dsr1": {
+      "gsm8k": 0.91
+    },
+    "dsv4": {
+      "aime26": 0.3,
+      "gpqa_diamond_cot_n_shot": 0.65,
+      "gsm8k": 0.91
+    },
+    "glm5": {
+      "gsm8k": 0.94
+    },
+    "glm5.1": {
+      "gsm8k": 0.93
+    },
+    "gptoss": {
+      "gsm8k": 0.91
+    },
+    "kimik2.5": {
+      "gsm8k": 0.9
+    },
+    "minimaxm2.5": {
+      "gsm8k": 0.92
+    },
+    "qwen3.5": {
+      "gsm8k": 0.94
+    }
+  }
+}

utils/evals/thresholds.yaml

@@ -186,7 +186,7 @@ def main() -> int:
     )
     parser.add_argument(
         "--thresholds", default=None,
-        help="Path to thresholds config, YAML or JSON (default: utils/evals/thresholds.yaml)",
+        help="Path to thresholds config, YAML or JSON (default: utils/evals/thresholds.json)",
     )
     parser.add_argument(
         "--meta-env", default="meta_env.json",
@@ -229,7 +229,7 @@ def main() -> int:
     config = {"default": {}, "models": {}}
     thresholds_path = args.thresholds
     if thresholds_path is None:
-        default_path = Path(__file__).parent / "thresholds.yaml"
+        default_path = Path(__file__).parent / "thresholds.json"
         if default_path.exists():
             thresholds_path = str(default_path)
     if thresholds_path: