|
| 1 | +{ |
| 2 | + "record": "Per-milestone BENCH compliance (VAL-GATE-015)", |
| 3 | + "contract": "library/perf-contract.md: median-of-5 per phase via git-workload-benchmark-multi.py; red iff >5% relative AND >10ms absolute vs the active baseline; single runs never gate; serialized on a quiet machine.", |
| 4 | + "baselines": { |
| 5 | + "m1": { |
| 6 | + "artifact": ".agents/benchmarks/restructure-baseline/medians-m1.json", |
| 7 | + "recorded_at_commit": "ae1e13f (benchmarked), 809d684 (artifact commit)", |
| 8 | + "status": "archived; superseded by the post-M3 rebaseline after three independent drift adjudications proved it brittle on this machine for diff/fsck/clone/checkout" |
| 9 | + }, |
| 10 | + "active_post_m3": { |
| 11 | + "artifact": ".agents/benchmarks/restructure-baseline/medians.json", |
| 12 | + "recorded_at_commit": "2b811f03493e861e5a7786592d25ac73b39e1aba", |
| 13 | + "recorded_at": "2026-07-04T01:14:37Z", |
| 14 | + "authorized_by": "orchestrator sign-off for misc-a-bench-rebaseline", |
| 15 | + "provenance": ".agents/benchmarks/restructure-baseline/provenance-post-m3.md" |
| 16 | + } |
| 17 | + }, |
| 18 | + "milestones": [ |
| 19 | + { |
| 20 | + "milestone": "M2 (dead-surface)", |
| 21 | + "runs": [ |
| 22 | + { |
| 23 | + "source": "m2-perf-triage-bench-red (HEAD 4e37c41, 2026-07-03)", |
| 24 | + "verdict": "in band after adjudication", |
| 25 | + "detail": "Stored-M1 compare red on fsck (+20.6ms/+11.0%; repeat +23.7ms/+12.7%). Rebuilt baseline code ae1e13f was red against its OWN stored artifact under identical conditions (diff +37.5%, fsck +11.5%), proving ambient drift, not a HEAD regression. Adjudication accepted by M2 scrutiny (PASS) and recorded in library/perf-contract.md." |
| 26 | + } |
| 27 | + ] |
| 28 | + }, |
| 29 | + { |
| 30 | + "milestone": "M3 (config/telemetry)", |
| 31 | + "runs": [ |
| 32 | + { |
| 33 | + "source": "m3-telemetry-registry feature BENCH", |
| 34 | + "verdict": "in band (0 red phases)", |
| 35 | + "detail": "checkout -18.0%, clone -9.7%, diff -53.7%, edit -3.9%, fsck -0.9%, read_search +1.9%, status -7.6%." |
| 36 | + }, |
| 37 | + { |
| 38 | + "source": "M3 user-testing VAL-CONF-013 (HEAD bc1753f)", |
| 39 | + "verdict": "in band after adjudication", |
| 40 | + "detail": "Moving red set across two stored-artifact runs (run1 clone +5.1%/fsck +8.3%; run2 diff/fsck/status); rebuilt ae1e13f sentinel red against its own artifact (clone +7.9%/+241ms, fsck +8.3%); direct HEAD-vs-sentinel showed HEAD equal or faster on the stored-red phases. Adjudicated noise; led to the orchestrator-signed post-M3 rebaseline (misc-a)." |
| 41 | + }, |
| 42 | + { |
| 43 | + "source": "misc-a-bench-rebaseline (post-M3 2b811f0)", |
| 44 | + "verdict": "baseline re-recorded", |
| 45 | + "detail": "Accepted run pair agreed 0-red in both directions; current-HEAD confirmation vs the new baseline: 0 red phases." |
| 46 | + } |
| 47 | + ] |
| 48 | + }, |
| 49 | + { |
| 50 | + "milestone": "M4a (core hot path)", |
| 51 | + "runs": [ |
| 52 | + { |
| 53 | + "source": "m4a-store-extraction feature BENCH", |
| 54 | + "verdict": "in band (0 red phases)", |
| 55 | + "detail": "checkout -14.2%, clone +2.2%, diff -41.5%, edit +1.0%, fsck +4.7%, read_search -9.5%, status -15.9%." |
| 56 | + }, |
| 57 | + { |
| 58 | + "source": "m4a-batcher-split feature BENCH (HEAD a687a84)", |
| 59 | + "verdict": "in band after adjudication", |
| 60 | + "detail": "Stored compare red on checkout/clone; rebuilt 2b811f0 sentinel equally red vs the same stored artifact under the same load; direct candidate-vs-sentinel: checkout +3.9ms, clone +3.9%, edit -0.8ms. Adjudicated ambient drift per perf-contract." |
| 61 | + }, |
| 62 | + { |
| 63 | + "source": "M4a user-testing VAL-CORE-008 (HEAD 9454886, load ~3.9)", |
| 64 | + "verdict": "in band (0 red phases)", |
| 65 | + "detail": "bench-compare exit 0; edit 5.2ms vs 5.0ms baseline (+3.8%, +0.2ms)." |
| 66 | + } |
| 67 | + ] |
| 68 | + }, |
| 69 | + { |
| 70 | + "milestone": "M4b (schema/overlay)", |
| 71 | + "runs": [ |
| 72 | + { |
| 73 | + "source": "m4b-overlay-directoryize feature BENCH", |
| 74 | + "verdict": "in band (0 red phases)", |
| 75 | + "detail": "bench-compare exit 0, 7 within band." |
| 76 | + }, |
| 77 | + { |
| 78 | + "source": "m4b-overlay-partial-store feature BENCH", |
| 79 | + "verdict": "in band (0 red phases)", |
| 80 | + "detail": "checkout +9.4ms stayed below the >10ms absolute red threshold; all other phases faster than baseline." |
| 81 | + }, |
| 82 | + { |
| 83 | + "source": "m4b-agentfs-directoryize feature BENCH", |
| 84 | + "verdict": "in band (0 red phases)", |
| 85 | + "detail": "clone -641.5ms / -22.9%; all other phases OK." |
| 86 | + }, |
| 87 | + { |
| 88 | + "source": "M4b user-testing VAL-CORE-018 (HEAD a62b0a1, load ~1.9-2.5)", |
| 89 | + "verdict": "in band (0 red phases)", |
| 90 | + "detail": "clone 2800.9ms -> 2741.3ms (-2.1%); diff +6.8ms and read_search +1.6ms sub-10ms absolute." |
| 91 | + } |
| 92 | + ] |
| 93 | + }, |
| 94 | + { |
| 95 | + "milestone": "M5 (crate split)", |
| 96 | + "runs": [ |
| 97 | + { |
| 98 | + "source": "m5-fuse-crate feature BENCH", |
| 99 | + "verdict": "in band (0 red phases)", |
| 100 | + "detail": "checkout 49.0ms, clone 2037.0ms, diff 31.1ms, edit 4.4ms, fsck 157.6ms, read_search 7.2ms, status 142.2ms." |
| 101 | + }, |
| 102 | + { |
| 103 | + "source": "m5-mount-crate feature BENCH", |
| 104 | + "verdict": "in band (0 red phases)", |
| 105 | + "detail": "checkout 43.1ms, clone 1872.2ms, diff 42.4ms, edit 3.7ms, fsck 180.3ms, read_search 7.7ms, status 121.1ms." |
| 106 | + }, |
| 107 | + { |
| 108 | + "source": "m5-fix-base-drift-coherence feature BENCH", |
| 109 | + "verdict": "in band (0 red phases)", |
| 110 | + "detail": "All candidate medians faster than the active baseline (checkout 50.1ms, clone 1879.4ms, diff 33.2ms, edit 3.0ms, fsck 169.8ms, read_search 6.5ms, status 118.5ms)." |
| 111 | + } |
| 112 | + ] |
| 113 | + }, |
| 114 | + { |
| 115 | + "milestone": "M6 (semantics)", |
| 116 | + "runs": [ |
| 117 | + { |
| 118 | + "source": "m6-fix-readdir-scaling feature BENCH", |
| 119 | + "verdict": "in band (0 red phases)", |
| 120 | + "detail": "checkout 54.6ms, clone 2605.7ms, diff 33.1ms, edit 4.5ms, fsck 206.4ms, read_search 8.1ms, status 172.3ms; bench-compare 0 red, 7 within band." |
| 121 | + } |
| 122 | + ] |
| 123 | + }, |
| 124 | + { |
| 125 | + "milestone": "M7 (cli/docs + attestation)", |
| 126 | + "runs": [ |
| 127 | + { |
| 128 | + "source": "m7-attestation round 1 (HEAD ff91302, quiet load 1.4-2.1)", |
| 129 | + "verdict": "red (checkout) — superseded by the round-3 closure", |
| 130 | + "detail": "Stored compare red on checkout only (56.7 -> 133.5ms, +135%); same-window sentinel did NOT clear it (sentinel 70.3ms vs HEAD 133.5ms), so it could not be adjudicated as noise; all other phases in band or faster. Deferred to a wall-time bisect." |
| 131 | + }, |
| 132 | + { |
| 133 | + "source": "m7-attestation round 2 wall-time bisect (HEAD 5535368, quiet load 2.0-3.5)", |
| 134 | + "verdict": "red reproduced; bisected to ea416c8 — superseded by the round-3 closure", |
| 135 | + "detail": "Checkout red reproduced (104.5ms vs same-window sentinel 77.2ms). Median-of-5 wall bisect c1937a3..3669c7b in a detached worktree, alternating probe order: first bad commit ea416c8 (scope run reads; fork before tokio). Checkout-phase FUSE profile counters identical parent-vs-culprit, locating the tax in the run-sandbox environment, not the FUSE hot path. Round 2 also established the brittle-phase repeat rule after a first-run diff red (+45%) evaporated on repeat." |
| 136 | + }, |
| 137 | + { |
| 138 | + "source": "m7-opt-sandbox-bringup closure (HEAD dae8aa6, quiet load 3.4-4.2)", |
| 139 | + "verdict": "in band (0 red phases) — checkout red CLOSED with zero code change", |
| 140 | + "detail": "Root cause: the bisect-era workload ran PATH git = the ~/.git-ai/bin/git shim inside the sandbox namespace; ea416c8's namespace pushed the shim's daemon/state logic onto a slow path per git call. dae8aa6 pins /usr/bin/git through the sandbox and harnesses, removing the tax from both legs. Official candidate/sentinel pair plus mandated brittle-phase repeat vs the stored post-M3 baseline: repeat 0 red, all 7 phases in band (checkout 56.7 -> 56.9ms +0.4%, clone +2.9%; the first candidate run's clone +24.3% red evaporated on the repeat). Same-window sentinel green; candidate checkout beat the sentinel by 17%. Exact-bench-shape zone attribution showed no measurable sandbox cost on any variant. Evidence: evidence/m7-cli-docs/m7-opt-sandbox-bringup/, library/perf-contract.md CLOSED entry." |
| 141 | + }, |
| 142 | + { |
| 143 | + "source": "m7-attestation final (attested HEAD 230c30a)", |
| 144 | + "verdict": "in band (0 red phases, cited pair)", |
| 145 | + "detail": "medians.json records the dae8aa6 official repeat pair vs the active post-M3 baseline: all 7 phases passed (checkout +0.4%, clone +2.9%, every other phase faster than baseline). Cited at 230c30a per the orchestrator round-3 ruling: the only commit after dae8aa6 changes cfg(target_os=macos) code, docs, and clap help text only — Linux binary perf semantics unchanged." |
| 146 | + } |
| 147 | + ] |
| 148 | + } |
| 149 | + ] |
| 150 | +} |
0 commit comments