Skip to content

Commit f8fe392

Browse files
claudeclaude
committed
feat(guard): pre-push benchmark + coverage regression guard
Adds a pre-push hook that runs before every git push: Benchmarks (go test -bench=. -benchmem -count=3): - Runs via cpulimit/nice at 85% CPU, -p=85% nproc - Compares ns/op against coverage/bench-baseline.txt using benchstat (if installed) or a built-in awk parser - Blocks push if any benchmark regresses > 20% (BENCH_REGRESS_PCT env) Coverage (go test -coverprofile -covermode=atomic): - Uses nice -n 15 (not cpulimit): cpulimit only throttles the parent go test process, not per-package children, causing partial profiles - Blocks push if total coverage drops > 1% (COVER_DROP_PCT env) - Baseline: 21.1% Both baselines are committed (bench-baseline.txt, coverage-baseline.txt). Use 'make bench-baseline' to reset after intentional perf/coverage changes. Fix: $NF → $$NF in Makefile awk to prevent make variable expansion. Fix: cpulimit not used for coverage (child process throttle issue). Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1 parent 5eb068f commit f8fe392

6 files changed

Lines changed: 1524 additions & 5 deletions

File tree

‎.githooks/pre-push‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
for check in "$(dirname "$0")"/push-checks/*.sh; do
3+
sh "$check" || exit 1
4+
done

‎.githooks/push-checks/01-bench-cover.sh‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/bin/sh
2+
# pre-push: benchmark regression + coverage drop guard.
3+
#
4+
# Baselines (committed, so shared across the team):
5+
# coverage/bench-baseline.txt — raw Go benchmark output (-count=3)
6+
# coverage/coverage-baseline.txt — total coverage percentage
7+
#
8+
# Thresholds (override via env):
9+
# BENCH_REGRESS_PCT — max allowed slowdown in ns/op (default: 20)
10+
# COVER_DROP_PCT — max allowed coverage drop in % (default: 1)
11+
#
12+
# Tools:
13+
# benchstat (golang.org/x/perf/cmd/benchstat) — used when installed.
14+
# Otherwise: built-in awk parser for ns/op comparison.
15+
# go tool cover — always available.
16+
#
17+
# To reset baselines after an intentional change:
18+
# make bench-baseline (or see instructions at end of this file)
19+
20+
REPO="$(git rev-parse --show-toplevel)"
21+
NCPU=$(nproc)
22+
P85=$(( NCPU * 85 / 100 ))
23+
[ "$P85" -lt 1 ] && P85=1
24+
25+
# CPU limiter
26+
if command -v cpulimit >/dev/null 2>&1; then
27+
LIMIT_PCT=$(( NCPU * 85 ))
28+
RUNNER="cpulimit -l ${LIMIT_PCT} --"
29+
else
30+
RUNNER="nice -n 15"
31+
fi
32+
33+
export CGO_ENABLED=0
34+
BENCH_REGRESS_PCT="${BENCH_REGRESS_PCT:-20}"
35+
COVER_DROP_PCT="${COVER_DROP_PCT:-1}"
36+
37+
BENCH_BASELINE="${REPO}/coverage/bench-baseline.txt"
38+
BENCH_CURRENT="${REPO}/coverage/bench-current.txt"
39+
COV_BASELINE="${REPO}/coverage/coverage-baseline.txt"
40+
COV_PROFILE="${REPO}/coverage/coverage.out"
41+
mkdir -p "${REPO}/coverage"
42+
43+
FAILED=0
44+
45+
# ── Benchmarks ──────────────────────────────────────────────────────────────
46+
47+
echo "pre-push: running benchmarks (count=3, cpu≤85%)..."
48+
$RUNNER go test -bench=. -benchmem -count=3 \
49+
-p "${P85}" ./... 2>/dev/null | \
50+
grep -v "^---" > "$BENCH_CURRENT"
51+
52+
if [ ! -s "$BENCH_CURRENT" ]; then
53+
echo "WARNING: no benchmark output produced — skipping bench guard."
54+
else
55+
if [ -f "$BENCH_BASELINE" ]; then
56+
echo "pre-push: comparing benchmarks against baseline..."
57+
58+
if command -v benchstat >/dev/null 2>&1; then
59+
# benchstat is the gold standard.
60+
BENCH_DIFF=$(benchstat "$BENCH_BASELINE" "$BENCH_CURRENT" 2>&1)
61+
echo "$BENCH_DIFF"
62+
# Detect regressions: lines with +XX.XX% where XX > threshold.
63+
REGRESSIONS=$(echo "$BENCH_DIFF" | awk -v thr="$BENCH_REGRESS_PCT" '
64+
/\+[0-9]+\.[0-9]+%/ {
65+
match($0, /\+([0-9]+\.[0-9]+)%/, m)
66+
if (m[1]+0 > thr+0) print $0
67+
}')
68+
if [ -n "$REGRESSIONS" ]; then
69+
echo "" >&2
70+
echo "PUSH BLOCKED: benchmark regression > ${BENCH_REGRESS_PCT}%:" >&2
71+
echo "$REGRESSIONS" >&2
72+
echo "" >&2
73+
echo "To reset: make bench-baseline" >&2
74+
FAILED=1
75+
fi
76+
else
77+
# Fallback: parse ns/op with awk and compare.
78+
REGRESSIONS=$(awk -v thr="$BENCH_REGRESS_PCT" '
79+
# Pass 1 (baseline): build name→ns map.
80+
NR==FNR && /^Benchmark/ {
81+
name = $1
82+
for (i=2; i<=NF; i++) {
83+
if ($(i) == "ns/op") { base[name] = $(i-1)+0; break }
84+
}
85+
next
86+
}
87+
# Pass 2 (current): compare.
88+
/^Benchmark/ {
89+
name = $1
90+
for (i=2; i<=NF; i++) {
91+
if ($(i) == "ns/op") {
92+
curr = $(i-1)+0
93+
if (name in base && base[name] > 0) {
94+
pct = (curr - base[name]) / base[name] * 100
95+
if (pct > thr+0) {
96+
printf " %s: +%.1f%% (%.1f → %.1f ns/op)\n",
97+
name, pct, base[name], curr
98+
}
99+
}
100+
break
101+
}
102+
}
103+
}
104+
' "$BENCH_BASELINE" "$BENCH_CURRENT")
105+
if [ -n "$REGRESSIONS" ]; then
106+
echo "" >&2
107+
echo "PUSH BLOCKED: benchmark regression > ${BENCH_REGRESS_PCT}% ns/op:" >&2
108+
echo "$REGRESSIONS" >&2
109+
echo "" >&2
110+
echo "To reset: make bench-baseline" >&2
111+
FAILED=1
112+
else
113+
echo "pre-push: benchmarks ok (no regression > ${BENCH_REGRESS_PCT}%)."
114+
fi
115+
fi
116+
else
117+
echo "pre-push: no benchmark baseline — recording now."
118+
fi
119+
120+
# Update baseline only when no regression was found.
121+
if [ "$FAILED" -eq 0 ]; then
122+
cp "$BENCH_CURRENT" "$BENCH_BASELINE"
123+
fi
124+
fi
125+
126+
# ── Coverage ─────────────────────────────────────────────────────────────────
127+
128+
echo "pre-push: running coverage..."
129+
# Use nice -n 15 (not cpulimit) for coverage: cpulimit only limits the main
130+
# go test process but not the per-package child binaries, which causes
131+
# go test to write only partial coverage data to the profile.
132+
nice -n 15 go test -timeout 300s -p "${P85}" -parallel "${P85}" \
133+
-coverprofile="$COV_PROFILE" -covermode=atomic \
134+
./... >/dev/null 2>&1
135+
136+
CURRENT_COV=$(go tool cover -func="$COV_PROFILE" 2>/dev/null | \
137+
awk '/^total:/ { gsub(/%/, "", $NF); printf "%.1f", $NF }')
138+
139+
if [ -z "$CURRENT_COV" ]; then
140+
echo "WARNING: could not determine coverage — skipping coverage guard."
141+
else
142+
echo "pre-push: coverage ${CURRENT_COV}%"
143+
if [ -f "$COV_BASELINE" ]; then
144+
BASELINE_COV=$(cat "$COV_BASELINE")
145+
DROPPED=$(awk -v curr="$CURRENT_COV" -v base="$BASELINE_COV" \
146+
-v thr="$COVER_DROP_PCT" \
147+
'BEGIN { print (base - curr > thr+0) ? "yes" : "no" }')
148+
if [ "$DROPPED" = "yes" ]; then
149+
echo "" >&2
150+
echo "PUSH BLOCKED: coverage dropped ${BASELINE_COV}% → ${CURRENT_COV}% (limit: -${COVER_DROP_PCT}%)." >&2
151+
echo "" >&2
152+
echo "To reset: echo ${CURRENT_COV} > coverage/coverage-baseline.txt" >&2
153+
FAILED=1
154+
else
155+
echo "pre-push: coverage ok (baseline: ${BASELINE_COV}%)."
156+
fi
157+
else
158+
echo "pre-push: no coverage baseline — recording now."
159+
fi
160+
161+
if [ "$FAILED" -eq 0 ]; then
162+
echo "$CURRENT_COV" > "$COV_BASELINE"
163+
fi
164+
fi
165+
166+
[ "$FAILED" -eq 0 ] && echo "pre-push: all guards passed."
167+
exit "$FAILED"

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ coverage.html
3636
coverage/*
3737
# Exception: benchmark baseline IS committed
3838
!coverage/bench-baseline.json
39+
!coverage/bench-baseline.txt
40+
!coverage/coverage-baseline.txt
3941
reference/mendix-repl/
4042
reference/mxbuild/
4143
reference/mendixmodellib/

‎Makefile‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ TEST_PARALLEL ?= $(_85PCT)
5151
# Hard ceiling on how long the full test suite may run.
5252
TEST_TIMEOUT ?= 180s
5353

54-
.PHONY: build build-debug release clean test test-mdl report report-bench report-reset-baseline grammar sync-skills sync-commands sync-lint-rules sync-changelog sync-examples sync-all docs documentation docs-site docs-serve source-tree sbom sbom-report lint lint-go fmt vet update-helpdesk-golden test-helpdesk-regression setup
54+
.PHONY: build build-debug release clean test _test-inner test-mdl report report-bench report-reset-baseline bench-baseline grammar sync-skills sync-commands sync-lint-rules sync-changelog sync-examples sync-all docs documentation docs-site docs-serve source-tree sbom sbom-report lint lint-go fmt vet update-helpdesk-golden test-helpdesk-regression setup
5555

5656
setup:
5757
git config core.hooksPath .githooks
@@ -205,19 +205,42 @@ report:
205205
--bench-diff coverage/bench-diff.txt \
206206
--out-html coverage/report.html
207207

208-
# Run only benchmarks and update the baseline
208+
# Record benchmark + coverage baselines used by the pre-push guard.
209+
# Run this after intentional perf changes to silence the guard.
210+
#
211+
# Benchmarks: cpulimit/nice wraps go test directly (single process per pkg).
212+
# Coverage: always uses nice -n 15; cpulimit only limits the parent
213+
# go test process, not the per-package child binaries, which
214+
# causes go test to write only partial coverage data.
215+
bench-baseline:
216+
@mkdir -p coverage
217+
@echo "Recording benchmark baseline (count=3, cpu≤85%)..."
218+
$(_CPU_RUNNER) go test -bench=. -benchmem -count=3 \
219+
-p $(_85PCT) ./... 2>/dev/null | grep -v "^---" > coverage/bench-baseline.txt
220+
@echo "Recording coverage baseline..."
221+
nice -n 15 go test -timeout 300s \
222+
-p $(_85PCT) -parallel $(_85PCT) \
223+
-coverprofile=coverage/coverage.out -covermode=atomic \
224+
./... >/dev/null 2>&1
225+
@go tool cover -func=coverage/coverage.out | \
226+
awk '/^total:/ { gsub(/%/,"",$$NF); printf "%.1f\n",$$NF }' > coverage/coverage-baseline.txt
227+
@echo "Benchmarks → coverage/bench-baseline.txt"
228+
@echo "Coverage → $$(cat coverage/coverage-baseline.txt)% (coverage/coverage-baseline.txt)"
229+
230+
# Run only benchmarks and update the baseline (legacy target)
209231
report-bench:
210232
@mkdir -p coverage
211233
CGO_ENABLED=0 go test -bench=. -benchmem -count=3 ./... > coverage/bench-results.txt
212234
@if command -v benchstat >/dev/null 2>&1; then \
213-
benchstat coverage/bench-baseline.json coverage/bench-results.txt > coverage/bench-diff.txt || true; \
235+
benchstat coverage/bench-baseline.txt coverage/bench-results.txt > coverage/bench-diff.txt || true; \
214236
cat coverage/bench-diff.txt; \
215237
fi
216238

217239
# Reset benchmark baseline (use after major refactors)
218240
report-reset-baseline:
219-
echo '[]' > coverage/bench-baseline.json
220-
@echo "Baseline reset."
241+
echo '' > coverage/bench-baseline.txt
242+
echo '' > coverage/coverage-baseline.txt
243+
@echo "Baselines reset."
221244

222245
# Check MDL syntax for all doctype example scripts
223246
check-mdl: build

0 commit comments

Comments
 (0)