fix: enforce EXCLUDED_FROM_GATE at check time and harden file reads

Author: clean6378-max-it

scripts/check_benchmark_regression.py

@@ -26,6 +26,8 @@ def load_results(results_path: str | Path) -> dict[str, float]:
     path = Path(results_path)
     try:
         data = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise BenchmarkDataError(f"cannot read {path}: {exc}") from exc
     except json.JSONDecodeError as exc:
         raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
     try:
@@ -57,6 +59,7 @@ def load_baseline_means(baselines_path: str | Path) -> dict[str, float]:
     path = Path(baselines_path)
     try:
         data = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
     except json.JSONDecodeError as exc:
         raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
     if not isinstance(data, dict):
@@ -97,6 +100,8 @@ def check_regression(
 
     failures: list[str] = []
     for name, base in baseline_means.items():
+        if name in EXCLUDED_FROM_GATE:
+            continue
         cur = flat.get(name)
         if cur is None:
             print(f"WARN: no current result for baseline {name!r}; skipping")
@@ -111,6 +116,8 @@ def check_regression(
             failures.append(name)
 
     for name in flat:
+        if name in EXCLUDED_FROM_GATE:
+            continue
         if name not in baseline_means:
             print(f"WARN: {name!r} has no baseline yet; not gated")
 

tests/test_check_benchmark_regression.py

@@ -13,6 +13,8 @@
     load_results,
 )
 
+GATED_BENCH = "test_parse_session_medium"
+
 
 def _write_results(path, benchmarks: list[dict]) -> None:
     path.write_text(
@@ -55,11 +57,11 @@ def test_regression_over_threshold_fails(tmp_path, capsys: pytest.CaptureFixture
     baselines = tmp_path / "baselines.json"
     _write_results(
         results,
-        [{"name": "test_parse_session_small", "stats": {"mean": 0.0002}}],
+        [{"name": GATED_BENCH, "stats": {"mean": 0.0025}}],
     )
     _write_baselines(
         baselines,
-        {"parse": {"test_parse_session_small": 0.0001}},
+        {"parse": {GATED_BENCH: 0.002}},
     )
 
     assert check_regression(results, baselines) == 1
@@ -72,11 +74,11 @@ def test_within_threshold_passes(tmp_path) -> None:
     baselines = tmp_path / "baselines.json"
     _write_results(
         results,
-        [{"name": "test_parse_session_small", "stats": {"mean": 0.00011}}],
+        [{"name": GATED_BENCH, "stats": {"mean": 0.0022}}],
     )
     _write_baselines(
         baselines,
-        {"parse": {"test_parse_session_small": 0.0001}},
+        {"parse": {GATED_BENCH: 0.002}},
     )
 
     assert check_regression(results, baselines) == 0
@@ -96,35 +98,58 @@ def test_load_results_requires_benchmarks_array(tmp_path) -> None:
         load_results(path)
 
 
+def test_load_results_rejects_missing_file(tmp_path) -> None:
+    with pytest.raises(BenchmarkDataError, match="cannot read"):
+        load_results(tmp_path / "missing.json")
+
+
 def test_zero_baseline_skips_ratio_check(tmp_path, capsys: pytest.CaptureFixture[str]) -> None:
     results = tmp_path / "results.json"
     baselines = tmp_path / "baselines.json"
     _write_results(
         results,
-        [{"name": "test_parse_session_small", "stats": {"mean": 0.0002}}],
+        [{"name": GATED_BENCH, "stats": {"mean": 0.0025}}],
     )
     _write_baselines(
         baselines,
-        {"parse": {"test_parse_session_small": 0.0}},
+        {"parse": {GATED_BENCH: 0.0}},
     )
 
     assert check_regression(results, baselines) == 0
-    assert "baseline for 'test_parse_session_small' is zero" in capsys.readouterr().out
+    assert f"baseline for '{GATED_BENCH}' is zero" in capsys.readouterr().out
 
 
 def test_exactly_at_threshold_passes(tmp_path) -> None:
     results = tmp_path / "results.json"
     baselines = tmp_path / "baselines.json"
     _write_results(
         results,
-        [{"name": "test_parse_session_small", "stats": {"mean": 0.00012}}],
+        [{"name": GATED_BENCH, "stats": {"mean": 0.0024}}],
+    )
+    _write_baselines(
+        baselines,
+        {"parse": {GATED_BENCH: 0.002}},
+    )
+
+    assert check_regression(results, baselines) == 0
+
+
+def test_excluded_benchmark_in_baselines_is_not_gated(
+    tmp_path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    results = tmp_path / "results.json"
+    baselines = tmp_path / "baselines.json"
+    _write_results(
+        results,
+        [{"name": "test_parse_session_small", "stats": {"mean": 0.001}}],
     )
     _write_baselines(
         baselines,
         {"parse": {"test_parse_session_small": 0.0001}},
     )
 
     assert check_regression(results, baselines) == 0
+    assert "REGRESSION" not in capsys.readouterr().out
 
 
 def test_missing_current_result_warns_without_failing(
@@ -135,7 +160,7 @@ def test_missing_current_result_warns_without_failing(
     _write_results(results, [])
     _write_baselines(
         baselines,
-        {"parse": {"test_parse_session_small": 0.0001}},
+        {"parse": {GATED_BENCH: 0.002}},
     )
 
     assert check_regression(results, baselines) == 0
@@ -148,7 +173,7 @@ def test_main_reports_benchmark_data_error(tmp_path, capsys: pytest.CaptureFixtu
     bad = tmp_path / "bad.json"
     bad.write_text("{}", encoding="utf-8")
     baselines = tmp_path / "baselines.json"
-    _write_baselines(baselines, {"parse": {"test_parse_session_small": 0.0001}})
+    _write_baselines(baselines, {"parse": {GATED_BENCH: 0.002}})
 
     assert main([str(bad), str(baselines)]) == 2
     assert "ERROR:" in capsys.readouterr().err