Skip to content

Commit 05bdb98

Browse files
scottgerringscottgerring
authored
ci: add continuous benchmark tracking dashboard (#3404)
1 parent 9bbce12 commit 05bdb98

2 files changed

Lines changed: 92 additions & 25 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 86 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,35 @@
1-
# This workflow runs a Criterion benchmark on a PR and compares the results against the base branch.
2-
# It is triggered on a PR or a push to main.
1+
# This workflow has two jobs:
2+
# 1. compareBenchmark: Runs on PRs with the "performance" label, comparing Criterion
3+
# benchmark results against the base branch using criterion-compare-action.
4+
# 2. continuousBenchmark: Runs daily on main via schedule, storing benchmark results
5+
# in the gh-pages branch and publishing a dashboard via github-action-benchmark.
6+
# Skips runs where the HEAD commit has already been benchmarked.
37
#
4-
# The workflow is gated on the presence of the "performance" label on the PR.
5-
#
6-
# The workflow runs on a self-hosted runner pool. We can't use the shared runners for this,
7-
# because they are only permitted to run on the default branch to preserve resources.
8-
#
9-
# In the future, we might like to consider using bencher.dev or the framework used by otel-golang here.
10-
on:
8+
# The PR job runs on shared GitHub runners to save resources.
9+
# The continuous job runs on a self-hosted bare-metal runner for consistent, accurate results.
10+
on:
1111
pull_request:
1212
types: [labeled, synchronize]
13-
push:
14-
branches:
15-
- main
16-
name: benchmark pull requests
13+
schedule:
14+
- cron: '0 6 * * *' # daily at 06:00 UTC
15+
workflow_dispatch:
16+
name: benchmark
1717
permissions:
1818
contents: read
1919
jobs:
20-
runBenchmark:
21-
name: run benchmark
20+
# ---------------------------------------------------------------------------
21+
# PR benchmark comparison
22+
# ---------------------------------------------------------------------------
23+
compareBenchmark:
24+
name: compare benchmarks (PR)
25+
if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'performance')
26+
runs-on: ubuntu-latest
2227
permissions:
2328
pull-requests: write
24-
25-
# If we're running on main, use our oracle bare-metal runner for accuracy.
26-
# If we're running on a PR, use github's shared workers to save resources.
27-
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'oracle-bare-metal-64cpu-512gb-x86-64' }}
28-
if: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'performance')) || github.event_name == 'push' }}
2929
container:
3030
image: rust:slim-bullseye
3131
env:
32-
# For PRs, compare against the base branch - e.g., 'main'.
33-
# For pushes to main, compare against the previous commit
34-
BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }}
32+
BRANCH_NAME: ${{ github.base_ref }}
3533
GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
3634
steps:
3735
- name: Harden the runner (Audit all outbound calls)
@@ -49,7 +47,7 @@ jobs:
4947

5048
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
5149
with:
52-
fetch-depth: 10 # Fetch a bit of history so we can do perf diffs
50+
fetch-depth: 10
5351

5452
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
5553
with:
@@ -58,3 +56,67 @@ jobs:
5856
- uses: boa-dev/criterion-compare-action@adfd3a94634fe2041ce5613eb7df09d247555b87 # v3.2.4
5957
with:
6058
branchName: ${{ env.BRANCH_NAME }}
59+
60+
# ---------------------------------------------------------------------------
61+
# Continuous benchmark tracking (daily schedule)
62+
# ---------------------------------------------------------------------------
63+
continuousBenchmark:
64+
name: continuous benchmark tracking
65+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
66+
# Use bare-metal runner on the upstream repo for consistent results; fall back to shared runners elsewhere
67+
runs-on: ${{ github.repository == 'open-telemetry/opentelemetry-rust' && 'oracle-bare-metal-64cpu-512gb-x86-64' || 'ubuntu-latest' }}
68+
permissions:
69+
contents: write
70+
container:
71+
image: rust:slim-bullseye
72+
env:
73+
GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
74+
steps:
75+
- name: Harden the runner (Audit all outbound calls)
76+
uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
77+
with:
78+
egress-policy: audit
79+
80+
- name: Setup container environment
81+
run: |
82+
apt-get update && apt-get install --fix-missing -y unzip cmake build-essential pkg-config curl git libssl-dev
83+
84+
- name: Make repo safe for Git inside container
85+
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
86+
87+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
88+
89+
- name: Check if commit already benchmarked
90+
id: check_duplicate
91+
run: |
92+
# Fetch the benchmark data file from gh-pages and see if this commit is already recorded
93+
DATA_URL="https://raw.githubusercontent.com/${{ github.repository }}/gh-pages/dev/bench/data.js"
94+
if curl -sf "$DATA_URL" | grep -q "${{ github.sha }}"; then
95+
echo "skip=true" >> "$GITHUB_OUTPUT"
96+
echo "Commit ${{ github.sha }} already benchmarked, skipping."
97+
else
98+
echo "skip=false" >> "$GITHUB_OUTPUT"
99+
fi
100+
101+
- uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
102+
if: steps.check_duplicate.outputs.skip != 'true'
103+
with:
104+
repo-token: ${{ secrets.GITHUB_TOKEN }}
105+
106+
- name: Run benchmarks
107+
if: steps.check_duplicate.outputs.skip != 'true'
108+
run: cargo bench --workspace --all-features -- --output-format bencher | tee output.txt
109+
110+
- name: Store benchmark result
111+
if: steps.check_duplicate.outputs.skip != 'true'
112+
uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073 # v1.21.0
113+
with:
114+
tool: 'cargo'
115+
output-file-path: output.txt
116+
github-token: ${{ secrets.GITHUB_TOKEN }}
117+
auto-push: true
118+
benchmark-data-dir-path: dev/bench
119+
# Alert if a benchmark regresses by more than 20%
120+
alert-threshold: '120%'
121+
comment-on-alert: true
122+
fail-on-alert: false

‎CONTRIBUTING.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,14 @@ issues.
321321
- Run `cargo test --all` - this will execute code and doc tests for all
322322
projects in this workspace.
323323
- Run `cargo bench` - this will run benchmarks to show performance
324-
- Run `cargo bench` - this will run benchmarks to show performance
325324
regressions
326325

326+
Benchmarks are run daily against `main` and results are tracked over time.
327+
The continuous benchmark dashboard is published at
328+
<https://open-telemetry.github.io/opentelemetry-rust/dev/bench/>.
329+
PRs with the `performance` label will also get a benchmark comparison
330+
comment showing any regressions or improvements.
331+
327332
## FAQ
328333

329334
### Where should I put third party propagators/exporters, contrib or standalone crates?

0 commit comments

Comments
 (0)