Skip to content

Commit f76e254

Browse files
committed
build: Improve benchmark tests
Signed-off-by: Paulo Gomes <pjbgf@linux.com>
1 parent 75f14f0 commit f76e254

2 files changed

Lines changed: 346 additions & 22 deletions

File tree

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Benchmarks
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
types: [opened, synchronize, reopened]
7+
8+
permissions: {}
9+
10+
jobs:
11+
benchmark:
12+
name: Compare Benchmarks
13+
runs-on: ubuntu-latest
14+
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
19+
steps:
20+
- name: Checkout PR branch
21+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
22+
with:
23+
path: new
24+
25+
- name: Checkout base branch
26+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
27+
with:
28+
ref: ${{ github.base_ref }}
29+
path: old
30+
31+
- name: Set up Go
32+
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
33+
with:
34+
go-version: stable
35+
cache-dependency-path: new/go.sum
36+
37+
- name: Install benchstat
38+
# renovate: datasource=go depName=golang.org/x/perf/cmd/benchstat
39+
run: go install golang.org/x/perf/cmd/benchstat@v0.0.0-20260211190930-8161c38c6cdc
40+
41+
- name: Run base branch benchmarks
42+
id: base-bench
43+
working-directory: old
44+
run: go test -run='^$' -bench=. -benchmem -count=1 ./test/... > ../base.txt 2>&1
45+
continue-on-error: true
46+
47+
- name: Run PR branch benchmarks
48+
id: pr-bench
49+
working-directory: new
50+
run: go test -run='^$' -bench=. -benchmem -count=1 ./test/... > ../pr.txt 2>&1
51+
continue-on-error: true
52+
53+
- name: Compare and report
54+
if: always()
55+
env:
56+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
57+
PR_NUMBER: ${{ github.event.pull_request.number }}
58+
REPO: ${{ github.repository }}
59+
BASE_OUTCOME: ${{ steps.base-bench.outcome }}
60+
PR_OUTCOME: ${{ steps.pr-bench.outcome }}
61+
run: |
62+
if [ "$BASE_OUTCOME" != "success" ] || [ "$PR_OUTCOME" != "success" ]; then
63+
{
64+
echo "## ⚠️ Benchmark Run Failed"
65+
echo ""
66+
echo "One or more benchmark runs did not complete successfully."
67+
echo "Please check the workflow logs for details."
68+
echo ""
69+
echo "| Step | Outcome |"
70+
echo "| --- | --- |"
71+
echo "| Base branch benchmarks | \`${BASE_OUTCOME}\` |"
72+
echo "| PR branch benchmarks | \`${PR_OUTCOME}\` |"
73+
} > comment.md
74+
else
75+
benchstat base.txt pr.txt > stat.txt 2>&1
76+
77+
# Extract lines with regressions greater than 5% on any metric.
78+
# Also capture infinite regressions (+∞ / +Inf) that appear when the
79+
# baseline measurement was zero.
80+
awk '
81+
/\+∞/ || /\+Inf/ { print; next }
82+
match($0, /\+[0-9]+\.?[0-9]*%/) {
83+
pct = substr($0, RSTART + 1, RLENGTH - 2)
84+
if (pct + 0 > 5) print
85+
}
86+
' stat.txt > regressions.txt
87+
88+
if [ -s regressions.txt ]; then
89+
{
90+
echo "## ⚠️ Benchmark Regressions Detected (>5%)"
91+
echo ""
92+
echo "The following benchmarks have degraded by more than 5% in time, memory, or allocations:"
93+
echo ""
94+
echo "\`\`\`"
95+
cat regressions.txt
96+
echo "\`\`\`"
97+
echo ""
98+
echo "<details>"
99+
echo "<summary>Full benchmark comparison (<code>benchstat</code>)</summary>"
100+
echo ""
101+
echo "\`\`\`"
102+
cat stat.txt
103+
echo "\`\`\`"
104+
echo "</details>"
105+
} > comment.md
106+
else
107+
{
108+
echo "## ✅ No Benchmark Regressions"
109+
echo ""
110+
echo "No benchmark regressions greater than 5% were detected."
111+
echo ""
112+
echo "<details>"
113+
echo "<summary>Full benchmark comparison (<code>benchstat</code>)</summary>"
114+
echo ""
115+
echo "\`\`\`"
116+
cat stat.txt
117+
echo "\`\`\`"
118+
echo "</details>"
119+
} > comment.md
120+
fi
121+
fi
122+
123+
gh pr comment "$PR_NUMBER" --edit-last --create-if-none --body-file comment.md --repo "$REPO"

0 commit comments

Comments
 (0)