tests/benchmarks/conftest.py for i in range(count): write_jsonl(project / f"session_{i:04d}.jsonl", 20) # Unique first_timestamp per session so export filenames do not collide in ZIP benches. first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z" @timon0305 timon0305 1 minute ago Member first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z" is only unique for count ≤ 120 (collides at lcm(24,60)). Add assert count <= 120 (or a wider format) so a future corpus bump can't produce colliding ZIP entries.

Author: clean6378-max-it

.gitignore

@@ -16,3 +16,4 @@ coverage/
 coverage.xml
 benchmark-results.json
 benchmarks/_raw.json
+benchmarks/_ci/

benchmarks/README.md

@@ -40,7 +40,7 @@ The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-js
 
 **Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory.
 
-**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These names are omitted from `baselines.json` when using `reduce_baselines.py`. Benchmarks without a baseline entry print a warning and do not fail the gate.
+**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These may appear in `baselines.json` for reference but are skipped by `check_benchmark_regression.py`. Benchmarks without a baseline entry print a warning and do not fail the gate.
 
 Missing gated benchmarks (renamed or removed tests still listed in `baselines.json`) fail the gate.
 

benchmarks/baselines.json

@@ -1,20 +1,24 @@
 {
-  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Values multiplied by 1.5× slack at seeding time. Excluded from gate (not in this file): test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).",
-  "updated": "2026-06-25T12:00:00Z",
+  "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR #97, run 28126772276). Excluded from gate (recorded for reference): test_parse_session_small, test_search_full_corpus (sub-ms CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).",
+  "updated": "2026-06-24T20:15:37Z",
   "machine": "Linux",
   "groups": {
     "parse": {
-      "test_parse_session_medium": 0.003037,
-      "test_parse_session_large": 0.030266,
+      "test_parse_session_small": 0.00010518068718225604,
+      "test_parse_session_medium": 0.002991333112179635,
+      "test_parse_session_large": 0.032311203818181436,
       "test_parse_large_peak_memory": 2032028.0
     },
     "export": {
-      "test_bulk_export_session_count[sessions-10]": 0.004253,
-      "test_bulk_export_session_count[sessions-50]": 0.021039,
-      "test_bulk_export_session_count[sessions-100]": 0.041709,
-      "test_bulk_export_zip_peak_memory[sessions-10]": 350546.0,
-      "test_bulk_export_zip_peak_memory[sessions-50]": 503331.0,
-      "test_bulk_export_zip_peak_memory[sessions-100]": 686874.0
+      "test_bulk_export_session_count[sessions-10]": 0.0042825538530803925,
+      "test_bulk_export_session_count[sessions-50]": 0.021406330209302382,
+      "test_bulk_export_session_count[sessions-100]": 0.04229194749999898,
+      "test_bulk_export_zip_peak_memory[sessions-10]": 350628.0,
+      "test_bulk_export_zip_peak_memory[sessions-50]": 506454.0,
+      "test_bulk_export_zip_peak_memory[sessions-100]": 694088.0
+    },
+    "search": {
+      "test_search_full_corpus": 0.0011120838654706596
     }
   }
 }

scripts/reduce_baselines.py

@@ -9,7 +9,6 @@
 from pathlib import Path
 
 from scripts.check_benchmark_regression import (
-    EXCLUDED_FROM_GATE,
     BenchmarkDataError,
     benchmark_entry_mean,
 )
@@ -59,25 +58,20 @@ def reduce_baselines(
                 f"{path} benchmarks[{index}] missing 'name' or measurable value"
             ) from exc
         bench_name = str(name)
-        if bench_name in EXCLUDED_FROM_GATE:
-            continue
         group = entry.get("group")
         if group not in GATED_GROUPS:
             continue
         groups[group][bench_name] = mean * slack
 
-    # Drop empty groups (e.g. search when only excluded benchmarks ran).
-    groups = {name: values for name, values in groups.items() if values}
-
     slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else ""
     machine_info = raw.get("machine_info")
     machine = machine_info.get("system") if isinstance(machine_info, dict) else None
     output: dict[str, object] = {
         "_note": (
             "Gated means from ubuntu-latest CI benchmark-results.json."
             f"{slack_note} "
-            "Excluded from gate (not written here): test_parse_session_small, "
-            "test_search_full_corpus (CI noise). "
+            "Excluded from gate (recorded for reference): test_parse_session_small, "
+            "test_search_full_corpus (sub-ms CI noise). "
             "Memory benchmarks use extra_info.peak_bytes (bytes); "
             "latency uses stats.mean (seconds)."
         ),

tests/benchmarks/conftest.py

@@ -102,7 +102,7 @@ def export_corpus(tmp_path: Path, request: pytest.FixtureRequest) -> Path:
     project.mkdir()
     for i in range(count):
         # Unique first_timestamp per session so export filenames do not collide in ZIP benches.
-        first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z"
+        first_ts = f"2026-06-12T{i // 60:02d}:{i % 60:02d}:00Z"
         write_jsonl(project / f"session_{i:04d}.jsonl", 20, first_timestamp=first_ts)
     return project
 

tests/test_reduce_baselines.py

@@ -38,12 +38,14 @@ def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None:
     output = reduce_baselines(raw, out)
 
     assert output["machine"] == "Linux"
+    assert set(output["groups"].keys()) == {"parse", "export", "search"}
     assert "test_parse_session_medium" in output["groups"]["parse"]
-    assert "test_parse_session_small" not in output["groups"]["parse"]
+    assert "test_parse_session_small" in output["groups"]["parse"]
+    assert output["groups"]["search"] == {}
     assert "cache" not in output["groups"]
 
 
-def test_reduce_baselines_skips_excluded_from_gate(tmp_path) -> None:
+def test_reduce_baselines_includes_search_benchmark(tmp_path) -> None:
     raw = tmp_path / "raw.json"
     out = tmp_path / "baselines.json"
     _write_raw(
@@ -56,8 +58,7 @@ def test_reduce_baselines_skips_excluded_from_gate(tmp_path) -> None:
 
     output = reduce_baselines(raw, out)
 
-    assert "search" not in output["groups"]
-    assert "test_search_full_corpus" not in json.loads(out.read_text(encoding="utf-8"))["groups"]
+    assert output["groups"]["search"]["test_search_full_corpus"] == pytest.approx(0.001)
 
 
 def test_reduce_baselines_applies_slack(tmp_path) -> None: