Skip to content

Commit 6a6cb6e

Browse files
fix: harden benchmark scripts against filesystem and type errors
Wrap reduce_baselines I/O in BenchmarkDataError; guard machine_info type; move float(mean) into try block in load_results; assert argparse exit code 2.
1 parent b4bf963 commit 6a6cb6e

3 files changed

Lines changed: 14 additions & 7 deletions

File tree

scripts/check_benchmark_regression.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ def load_results(results_path: str | Path) -> dict[str, float]:
3333
raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object")
3434
try:
3535
name = entry["name"]
36-
mean = entry["stats"]["mean"]
37-
except (KeyError, TypeError) as exc:
36+
mean = float(entry["stats"]["mean"])
37+
except (KeyError, TypeError, ValueError) as exc:
3838
raise BenchmarkDataError(
3939
f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'"
4040
) from exc
41-
results[str(name)] = float(mean)
41+
results[str(name)] = mean
4242
return results
4343

4444

scripts/reduce_baselines.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ def reduce_baselines(
3131
raw = json.loads(path.read_text(encoding="utf-8"))
3232
except json.JSONDecodeError as exc:
3333
raise BenchmarkDataError(f"invalid JSON in {path}: {exc}") from exc
34+
except OSError as exc:
35+
raise BenchmarkDataError(f"cannot read {path}: {exc}") from exc
3436

3537
try:
3638
entries = raw["benchmarks"]
@@ -55,15 +57,19 @@ def reduce_baselines(
5557
continue
5658
groups[group][str(name)] = mean * slack
5759

58-
machine_info = raw.get("machine_info", {})
60+
machine_info = raw.get("machine_info")
61+
machine = machine_info.get("system") if isinstance(machine_info, dict) else None
5962
output: dict[str, object] = {
6063
"_note": "CI gates the ubuntu benchmarks job when mean exceeds baseline by >20%.",
6164
"updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
62-
"machine": machine_info.get("system"),
65+
"machine": machine,
6366
"groups": groups,
6467
}
6568
out = Path(out_path)
66-
out.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
69+
try:
70+
out.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
71+
except OSError as exc:
72+
raise BenchmarkDataError(f"cannot write {out}: {exc}") from exc
6773
return output
6874

6975

tests/test_reduce_baselines.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,5 +74,6 @@ def test_reduce_baselines_cli_rejects_non_positive_slack(tmp_path) -> None:
7474
[{"group": "parse", "name": "test_parse_session_small", "stats": {"mean": 0.0001}}],
7575
)
7676

77-
with pytest.raises(SystemExit):
77+
with pytest.raises(SystemExit) as exc_info:
7878
main([str(raw), str(tmp_path / "out.json"), "--slack", "0"])
79+
assert exc_info.value.code == 2

0 commit comments

Comments
 (0)