Skip to content

Commit 684be43

Browse files
committed
build: Improve benchmark tests
Signed-off-by: Paulo Gomes <pjbgf@linux.com>
1 parent 75f14f0 commit 684be43

2 files changed

Lines changed: 358 additions & 22 deletions

File tree

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
name: Benchmarks
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
pr_number:
7+
description: 'Pull Request number'
8+
required: true
9+
type: number
10+
11+
permissions: {}
12+
13+
jobs:
14+
benchmark:
15+
name: Compare Benchmarks
16+
runs-on: ubuntu-latest
17+
18+
permissions:
19+
contents: read
20+
pull-requests: write
21+
22+
steps:
23+
- name: Get PR base ref
24+
id: pr-info
25+
env:
26+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
27+
run: |
28+
base_ref=$(gh pr view ${{ inputs.pr_number }} --repo ${{ github.repository }} --json baseRefName --jq .baseRefName)
29+
echo "base_ref=$base_ref" >> $GITHUB_OUTPUT
30+
31+
- name: Checkout PR branch
32+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
33+
with:
34+
ref: refs/pull/${{ inputs.pr_number }}/head
35+
path: new
36+
37+
- name: Checkout base branch
38+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
39+
with:
40+
ref: ${{ steps.pr-info.outputs.base_ref }}
41+
path: old
42+
43+
- name: Set up Go
44+
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
45+
with:
46+
go-version: stable
47+
cache-dependency-path: new/go.sum
48+
49+
- name: Install benchstat
50+
# renovate: datasource=go depName=golang.org/x/perf/cmd/benchstat
51+
run: go install golang.org/x/perf/cmd/benchstat@v0.0.0-20260211190930-8161c38c6cdc
52+
53+
- name: Run base branch benchmarks
54+
id: base-bench
55+
working-directory: old
56+
run: go test -run='^$' -bench=. -benchmem -count=1 ./test/... > ../base.txt 2>&1
57+
continue-on-error: true
58+
59+
- name: Run PR branch benchmarks
60+
id: pr-bench
61+
working-directory: new
62+
run: go test -run='^$' -bench=. -benchmem -count=1 ./test/... > ../pr.txt 2>&1
63+
continue-on-error: true
64+
65+
- name: Compare and report
66+
if: always()
67+
env:
68+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
69+
PR_NUMBER: ${{ inputs.pr_number }}
70+
REPO: ${{ github.repository }}
71+
BASE_OUTCOME: ${{ steps.base-bench.outcome }}
72+
PR_OUTCOME: ${{ steps.pr-bench.outcome }}
73+
run: |
74+
if [ "$BASE_OUTCOME" != "success" ] || [ "$PR_OUTCOME" != "success" ]; then
75+
{
76+
echo "## ⚠️ Benchmark Run Failed"
77+
echo ""
78+
echo "One or more benchmark runs did not complete successfully."
79+
echo "Please check the workflow logs for details."
80+
echo ""
81+
echo "| Step | Outcome |"
82+
echo "| --- | --- |"
83+
echo "| Base branch benchmarks | \`${BASE_OUTCOME}\` |"
84+
echo "| PR branch benchmarks | \`${PR_OUTCOME}\` |"
85+
} > comment.md
86+
else
87+
benchstat base.txt pr.txt > stat.txt 2>&1
88+
89+
# Extract lines with regressions greater than 5% on any metric.
90+
# Also capture infinite regressions (+∞ / +Inf) that appear when the
91+
# baseline measurement was zero.
92+
awk '
93+
/\+∞/ || /\+Inf/ { print; next }
94+
match($0, /\+[0-9]+\.?[0-9]*%/) {
95+
pct = substr($0, RSTART + 1, RLENGTH - 2)
96+
if (pct + 0 > 5) print
97+
}
98+
' stat.txt > regressions.txt
99+
100+
if [ -s regressions.txt ]; then
101+
{
102+
echo "## ⚠️ Benchmark Regressions Detected (>5%)"
103+
echo ""
104+
echo "The following benchmarks have degraded by more than 5% in time, memory, or allocations:"
105+
echo ""
106+
echo "\`\`\`"
107+
cat regressions.txt
108+
echo "\`\`\`"
109+
echo ""
110+
echo "<details>"
111+
echo "<summary>Full benchmark comparison (<code>benchstat</code>)</summary>"
112+
echo ""
113+
echo "\`\`\`"
114+
cat stat.txt
115+
echo "\`\`\`"
116+
echo "</details>"
117+
} > comment.md
118+
else
119+
{
120+
echo "## ✅ No Benchmark Regressions"
121+
echo ""
122+
echo "No benchmark regressions greater than 5% were detected."
123+
echo ""
124+
echo "<details>"
125+
echo "<summary>Full benchmark comparison (<code>benchstat</code>)</summary>"
126+
echo ""
127+
echo "\`\`\`"
128+
cat stat.txt
129+
echo "\`\`\`"
130+
echo "</details>"
131+
} > comment.md
132+
fi
133+
fi
134+
135+
gh pr comment "$PR_NUMBER" --edit-last --create-if-none --body-file comment.md --repo "$REPO"

0 commit comments

Comments
 (0)