fix(evals): allow long reasoning contexts

Author: adibarra

benchmarks/benchmark_lib.sh

@@ -757,7 +757,7 @@ except Exception:
 }
 
 # Compute the context length for eval-only mode.
-# Uses the requested benchmark context capped at the model's native max.
+# Eval-only runs use at least EVAL_MIN_MODEL_LEN, capped at the model's native max.
 # Sets EVAL_MAX_MODEL_LEN (needed by run_lm_eval).
 # Echoes the computed value for scripts to capture.
 #
@@ -769,6 +769,15 @@ compute_eval_context_length() {
     native_max=$(get_native_max_context_length "$model")
     native_max="${native_max:-0}"
 
+    # Fixed-sequence benchmark windows are often too short for reasoning evals.
+    # In eval-only mode, raise the context budget without changing recipe configs
+    # or throughput runs.
+    local eval_min="${EVAL_MIN_MODEL_LEN:-32768}"
+    if [ "${EVAL_ONLY:-false}" = "true" \
+        ] && [ "$benchmark_ctx" -lt "$eval_min" ] 2>/dev/null; then
+        benchmark_ctx="$eval_min"
+    fi
+
     if [ "$benchmark_ctx" -eq 0 ] 2>/dev/null; then
         benchmark_ctx="${native_max:-0}"
     fi

utils/evals/EVALS.md

@@ -151,7 +151,8 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]'
 | `EVAL_FRAMEWORK` | `lm-eval` | Eval framework to use |
 | `EVAL_TASKS_DIR` | `gsm8k,gpqa_diamond_cot_n_shot,aime26` | Comma-separated lm-eval task names, or one task YAML path |
 | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results |
-| `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) |
+| `EVAL_MAX_MODEL_LEN` | computed | Max context passed to the eval client and, where supported, the serving engine |
+| `EVAL_MIN_MODEL_LEN` | `32768` | Minimum context for eval-only reasoning tasks; capped at the model's native maximum and does not affect throughput runs |
 | `EVAL_CONCURRENT_REQUESTS` | `64` | Concurrent requests during eval; a space-separated list enables sequential batched evals against one live engine |
 | `EVAL_LIMIT` | empty | Limit eval to first N instances (smoke tests); empty = full set |
 

utils/evals/test_batched_eval.py

@@ -190,6 +190,44 @@ def test_workflow_concurrencies_are_independent_of_eval_metadata(
     assert any("missing result files for concurrency: 1, 4" in error for error in errors)
 
 
+def test_eval_context_has_reasoning_floor_capped_by_native_max(
+    tmp_path: Path,
+) -> None:
+    benchmark_lib = (
+        Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh"
+    )
+    script = r'''
+source "$BENCHMARK_LIB"
+get_native_max_context_length() { echo "$NATIVE_MAX"; }
+
+export EVAL_ONLY=true
+export EVAL_MIN_MODEL_LEN=32768
+export NATIVE_MAX=100000
+printf 'eval=%s\n' "$(compute_eval_context_length test-model 9472)"
+
+export NATIVE_MAX=16000
+printf 'native-cap=%s\n' "$(compute_eval_context_length test-model 9472)"
+
+export EVAL_ONLY=false
+export NATIVE_MAX=100000
+printf 'throughput=%s\n' "$(compute_eval_context_length test-model 9472)"
+'''
+    result = subprocess.run(
+        ["bash", "-c", script],
+        cwd=tmp_path,
+        env={**os.environ, "BENCHMARK_LIB": str(benchmark_lib)},
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.stdout.splitlines() == [
+        "eval=32768",
+        "native-cap=16000",
+        "throughput=9472",
+    ]
+
+
 def test_validate_scores_checks_threshold_for_every_concurrency(
     tmp_path: Path,
     monkeypatch,