Skip to content

Commit 29ad5a7

Browse files
fix(bench): harden reduce_baselines and fix Python 3.10 CI
1 parent 87a747c commit 29ad5a7

3 files changed

Lines changed: 34 additions & 5 deletions

File tree

benchmarks/baselines.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5\u00d7 slack at generation time. Refresh after intentional speedups via reduce_baselines.py.",
2+
"_note": "Gated means from ubuntu-latest CI benchmark-results.json. Values multiplied by 1.5x slack at generation time. Refresh after intentional speedups via reduce_baselines.py.",
33
"updated": "2026-06-25T21:48:35Z",
44
"machine": "Linux",
55
"groups": {

scripts/reduce_baselines.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import json
77
import math
88
import sys
9-
from datetime import UTC, datetime
9+
from datetime import datetime, timezone
1010
from pathlib import Path
1111

1212
_REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -75,6 +75,11 @@ def reduce_baselines(
7575
f"{path} benchmarks[{index}] ({bench_name!r}) has unknown group {group!r}; "
7676
f"expected one of {GATED_GROUPS}"
7777
)
78+
if bench_name in groups[group]:
79+
raise BenchmarkDataError(
80+
f"{path} benchmarks[{index}] ({raw_name!r}) duplicates normalized "
81+
f"benchmark {group!r}/{bench_name!r}"
82+
)
7883
groups[group][bench_name] = mean * slack
7984

8085
excluded = ", ".join(sorted(EXCLUDED_FROM_GATE))
@@ -83,7 +88,7 @@ def reduce_baselines(
8388
if excluded
8489
else ""
8590
)
86-
slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else ""
91+
slack_note = f" Values multiplied by {slack}x slack at generation time." if slack != 1.0 else ""
8792
machine_info = raw.get("machine_info")
8893
machine = machine_info.get("system") if isinstance(machine_info, dict) else None
8994
source_labels = {
@@ -97,7 +102,7 @@ def reduce_baselines(
97102
f"{slack_note}{excluded_note} "
98103
"Refresh after intentional speedups via reduce_baselines.py."
99104
),
100-
"updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
105+
"updated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
101106
"machine": machine,
102107
"groups": groups,
103108
}

tests/test_reduce_baselines.py

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,10 @@ def test_reduce_baselines_groups_and_slack(tmp_path) -> None:
5959
assert groups["parse"]["test_list_workspace_projects_nocache[composers-50]"] == pytest.approx(0.075)
6060
assert groups["export"]["test_post_export_zip[composers-10]"] == pytest.approx(0.015)
6161
assert groups["search"]["test_search_full_corpus"] == pytest.approx(0.06)
62+
assert groups["summary-cache"]["test_summary_cache_lookup[hit]"] == pytest.approx(0.00015)
6263
assert data["machine"] == "Linux"
6364
assert "ubuntu-latest CI benchmark-results.json" in data["_note"]
64-
assert "1. slack" in data["_note"]
65+
assert "1.5x slack" in data["_note"]
6566
assert output["groups"] == groups
6667

6768

@@ -121,6 +122,29 @@ def test_reduce_baselines_rejects_missing_group(tmp_path) -> None:
121122
reduce_baselines(raw, out)
122123

123124

125+
def test_reduce_baselines_rejects_duplicate_normalized_name(tmp_path) -> None:
126+
raw = tmp_path / "raw.json"
127+
out = tmp_path / "baselines.json"
128+
_write_raw(
129+
raw,
130+
[
131+
{
132+
"name": "test_summary_cache_lookup[hit]",
133+
"group": "summary-cache",
134+
"stats": {"mean": 0.0001},
135+
},
136+
{
137+
"name": "tests/benchmarks/test_summary_cache_bench.py::test_summary_cache_lookup[hit]",
138+
"group": "summary-cache",
139+
"stats": {"mean": 0.0002},
140+
},
141+
],
142+
)
143+
144+
with pytest.raises(BenchmarkDataError, match="duplicates normalized"):
145+
reduce_baselines(raw, out)
146+
147+
124148
def test_positive_float_rejects_non_finite() -> None:
125149
import argparse
126150

0 commit comments

Comments
 (0)