Skip to content

Commit 0226100

Browse files
mborlandmborland
authored
Merge pull request #220 from cppalliance/bench_docs
Add benchmarks workflow
2 parents 0f962d3 + 8f5c87c commit 0226100

41 files changed

Lines changed: 73820 additions & 375 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/benchmarks.yml‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
# ------------------------------------------------------------------------------
2+
# Copyright Matt Borland 2026.
3+
# Distributed under the Boost Software License,
4+
# Version 1.0. (See accompanying file LICENSE_1_0.txt
5+
# or copy at http://www.boost.org/LICENSE_1_0.txt)
6+
# ------------------------------------------------------------------------------
7+
#
8+
# Runs the Boost.SafeNumbers Google Benchmark suites (test/benchmarks/*.cpp) in
9+
# release mode across a spread of native runners. The benchmarks are built via
10+
# CMake with -DBOOST_SAFE_NUMBERS_BUILD_BENCHMARKS=ON, which pulls Google
11+
# Benchmark in with FetchContent. Each job emits per-suite JSON and a ready to
12+
# paste AsciiDoc section (produced by test/benchmarks/render_results.py) as
13+
# artifacts for the documentation benchmarks page.
14+
15+
name: Run Benchmarks
16+
17+
on:
18+
push:
19+
branches:
20+
- master
21+
- develop
22+
- feature/**
23+
pull_request:
24+
types: [opened, synchronize, reopened]
25+
workflow_dispatch:
26+
27+
concurrency:
28+
group: ${{ format('{0}:{1}:benchmarks', github.repository, github.ref) }}
29+
cancel-in-progress: true
30+
31+
env:
32+
GIT_FETCH_JOBS: 8
33+
# Repetitions plus the median aggregate keep the reported numbers stable; the
34+
# report-aggregates-only flag keeps the JSON small.
35+
BENCH_ARGS: "--benchmark_repetitions=5 --benchmark_min_time=0.10s --benchmark_report_aggregates_only=true --benchmark_out_format=json"
36+
37+
jobs:
38+
linux:
39+
name: ${{ matrix.title }}
40+
runs-on: ${{ matrix.os }}
41+
strategy:
42+
fail-fast: false
43+
matrix:
44+
include:
45+
- title: Linux x64
46+
os: ubuntu-latest
47+
arch: x64
48+
anchor: linux_x64
49+
desc: "Run on the GitHub Actions `ubuntu-latest` runner using GCC 14 in release mode (`-O2`, pass:[C++]20)."
50+
multilib: false
51+
cmake_extra: ""
52+
- title: Linux x86 (32-bit)
53+
os: ubuntu-latest
54+
arch: x86-32
55+
anchor: linux_x86_32
56+
desc: "Run on the GitHub Actions `ubuntu-latest` runner using GCC 14 targeting 32-bit x86 (`-m32`) in release mode (`-O2`, pass:[C++]20)."
57+
multilib: true
58+
cmake_extra: "-DCMAKE_CXX_FLAGS=-m32 -DCMAKE_C_FLAGS=-m32"
59+
- title: Linux ARM64
60+
os: ubuntu-24.04-arm
61+
arch: arm64
62+
anchor: linux_arm64
63+
desc: "Run on the GitHub Actions `ubuntu-24.04-arm` runner using GCC 14 in release mode (`-O2`, pass:[C++]20)."
64+
multilib: false
65+
cmake_extra: ""
66+
defaults:
67+
run:
68+
shell: bash
69+
steps:
70+
- uses: actions/checkout@v6
71+
- name: Install packages
72+
run: |
73+
sudo apt-get update
74+
sudo apt-get install -y g++-14 cmake
75+
if [ "${{ matrix.multilib }}" = "true" ]; then
76+
sudo apt-get install -y g++-14-multilib
77+
fi
78+
- name: Setup Boost
79+
run: |
80+
LIBRARY=${GITHUB_REPOSITORY#*/}
81+
echo "LIBRARY=$LIBRARY" >> $GITHUB_ENV
82+
REF=${GITHUB_BASE_REF:-$GITHUB_REF}
83+
REF=${REF#refs/heads/}
84+
BOOST_BRANCH=develop && [ "$REF" = "master" ] && BOOST_BRANCH=master || true
85+
cd ..
86+
git clone -b "$BOOST_BRANCH" --depth 1 "https://github.com/boostorg/boost.git" boost-root
87+
cd boost-root
88+
mkdir -p libs/$LIBRARY
89+
# Exclude doc/: it carries an Antora examples symlink (which git-bash on
90+
# Windows cannot recreate) and a large node_modules tree, neither of
91+
# which the benchmark build or depinst need.
92+
shopt -s extglob
93+
cp -r "$GITHUB_WORKSPACE"/!(doc) libs/$LIBRARY/
94+
git submodule update --init tools/boostdep
95+
python3 tools/boostdep/depinst/depinst.py --git_args "--jobs $GIT_FETCH_JOBS" $LIBRARY
96+
- name: Configure and build benchmarks
97+
run: |
98+
cd ../boost-root
99+
cmake -S . -B __build__ \
100+
-DCMAKE_BUILD_TYPE=Release \
101+
-DBUILD_TESTING=OFF \
102+
-DBOOST_INCLUDE_LIBRARIES="$LIBRARY;safe_numerics;random" \
103+
-DBOOST_SAFE_NUMBERS_BUILD_BENCHMARKS=ON \
104+
-DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_C_COMPILER=gcc-14 \
105+
${{ matrix.cmake_extra }}
106+
cmake --build __build__ --config Release --parallel 4 \
107+
--target benchmark_unsigned_operations benchmark_signed_operations benchmark_float_operations
108+
- name: Run benchmarks and render results
109+
run: |
110+
cd ../boost-root
111+
OUT="$GITHUB_WORKSPACE/bench-results"
112+
mkdir -p "$OUT"
113+
find_exe() { find __build__ -type f \( -name "$1" -o -name "$1.exe" \) | head -1; }
114+
"$(find_exe benchmark_unsigned_operations)" $BENCH_ARGS --benchmark_out="$OUT/unsigned.json"
115+
"$(find_exe benchmark_signed_operations)" $BENCH_ARGS --benchmark_out="$OUT/signed.json"
116+
"$(find_exe benchmark_float_operations)" $BENCH_ARGS --benchmark_out="$OUT/float.json"
117+
python3 "$GITHUB_WORKSPACE/test/benchmarks/render_results.py" \
118+
--title "${{ matrix.title }}" --anchor "${{ matrix.anchor }}" --desc "${{ matrix.desc }}" \
119+
--unsigned "$OUT/unsigned.json" --signed "$OUT/signed.json" --float "$OUT/float.json" \
120+
> "$OUT/section.adoc"
121+
echo "==================== ${{ matrix.title }} ===================="
122+
cat "$OUT/section.adoc"
123+
- name: Upload results
124+
if: always()
125+
uses: actions/upload-artifact@v6
126+
with:
127+
name: benchmarks-linux-${{ matrix.arch }}
128+
path: bench-results
129+
if-no-files-found: warn
130+
131+
macos:
132+
name: macOS ARM64
133+
runs-on: macos-latest
134+
defaults:
135+
run:
136+
shell: bash
137+
steps:
138+
- uses: actions/checkout@v6
139+
- name: Setup Boost
140+
run: |
141+
LIBRARY=${GITHUB_REPOSITORY#*/}
142+
echo "LIBRARY=$LIBRARY" >> $GITHUB_ENV
143+
REF=${GITHUB_BASE_REF:-$GITHUB_REF}
144+
REF=${REF#refs/heads/}
145+
BOOST_BRANCH=develop && [ "$REF" = "master" ] && BOOST_BRANCH=master || true
146+
cd ..
147+
git clone -b "$BOOST_BRANCH" --depth 1 "https://github.com/boostorg/boost.git" boost-root
148+
cd boost-root
149+
mkdir -p libs/$LIBRARY
150+
# Exclude doc/: it carries an Antora examples symlink (which git-bash on
151+
# Windows cannot recreate) and a large node_modules tree, neither of
152+
# which the benchmark build or depinst need.
153+
shopt -s extglob
154+
cp -r "$GITHUB_WORKSPACE"/!(doc) libs/$LIBRARY/
155+
git submodule update --init tools/boostdep
156+
python3 tools/boostdep/depinst/depinst.py --git_args "--jobs $GIT_FETCH_JOBS" $LIBRARY
157+
- name: Configure and build benchmarks
158+
run: |
159+
cd ../boost-root
160+
cmake -S . -B __build__ \
161+
-DCMAKE_BUILD_TYPE=Release \
162+
-DBUILD_TESTING=OFF \
163+
-DBOOST_INCLUDE_LIBRARIES="$LIBRARY;safe_numerics;random" \
164+
-DBOOST_SAFE_NUMBERS_BUILD_BENCHMARKS=ON
165+
cmake --build __build__ --config Release --parallel 4 \
166+
--target benchmark_unsigned_operations benchmark_signed_operations benchmark_float_operations
167+
- name: Run benchmarks and render results
168+
run: |
169+
cd ../boost-root
170+
OUT="$GITHUB_WORKSPACE/bench-results"
171+
mkdir -p "$OUT"
172+
find_exe() { find __build__ -type f \( -name "$1" -o -name "$1.exe" \) | head -1; }
173+
"$(find_exe benchmark_unsigned_operations)" $BENCH_ARGS --benchmark_out="$OUT/unsigned.json"
174+
"$(find_exe benchmark_signed_operations)" $BENCH_ARGS --benchmark_out="$OUT/signed.json"
175+
"$(find_exe benchmark_float_operations)" $BENCH_ARGS --benchmark_out="$OUT/float.json"
176+
python3 "$GITHUB_WORKSPACE/test/benchmarks/render_results.py" \
177+
--title "macOS ARM64" --anchor "macos_arm64" \
178+
--desc "Run on the GitHub Actions \`macos-latest\` runner (Apple Silicon) using Apple Clang in release mode (\`-O2\`, pass:[C++]20)." \
179+
--unsigned "$OUT/unsigned.json" --signed "$OUT/signed.json" --float "$OUT/float.json" \
180+
> "$OUT/section.adoc"
181+
echo "==================== macOS ARM64 ===================="
182+
cat "$OUT/section.adoc"
183+
- name: Upload results
184+
if: always()
185+
uses: actions/upload-artifact@v6
186+
with:
187+
name: benchmarks-macos-arm64
188+
path: bench-results
189+
if-no-files-found: warn
190+
191+
windows:
192+
name: ${{ matrix.title }}
193+
runs-on: ${{ matrix.os }}
194+
strategy:
195+
fail-fast: false
196+
matrix:
197+
include:
198+
- title: Windows x64
199+
os: windows-latest
200+
arch: x64
201+
anchor: windows_x64
202+
cmake_arch: x64
203+
desc: "Run on the GitHub Actions `windows-latest` runner using MSVC in release mode (`/O2`, pass:[C++]20)."
204+
- title: Windows x86 (32-bit)
205+
os: windows-latest
206+
arch: x86-32
207+
anchor: windows_x86_32
208+
cmake_arch: Win32
209+
desc: "Run on the GitHub Actions `windows-latest` runner using MSVC targeting 32-bit x86 in release mode (`/O2`, pass:[C++]20)."
210+
- title: Windows ARM64
211+
os: windows-11-arm
212+
arch: arm64
213+
anchor: windows_arm64
214+
cmake_arch: ARM64
215+
desc: "Run on the GitHub Actions `windows-11-arm` runner using MSVC in release mode (`/O2`, pass:[C++]20)."
216+
defaults:
217+
run:
218+
shell: bash
219+
steps:
220+
- uses: actions/checkout@v6
221+
- name: Setup Boost
222+
run: |
223+
LIBRARY=${GITHUB_REPOSITORY#*/}
224+
echo "LIBRARY=$LIBRARY" >> $GITHUB_ENV
225+
REF=${GITHUB_BASE_REF:-$GITHUB_REF}
226+
REF=${REF#refs/heads/}
227+
BOOST_BRANCH=develop && [ "$REF" = "master" ] && BOOST_BRANCH=master || true
228+
cd ..
229+
git clone -b "$BOOST_BRANCH" --depth 1 "https://github.com/boostorg/boost.git" boost-root
230+
cd boost-root
231+
mkdir -p libs/$LIBRARY
232+
# Exclude doc/: it carries an Antora examples symlink (which git-bash on
233+
# Windows cannot recreate) and a large node_modules tree, neither of
234+
# which the benchmark build or depinst need.
235+
shopt -s extglob
236+
cp -r "$GITHUB_WORKSPACE"/!(doc) libs/$LIBRARY/
237+
git submodule update --init tools/boostdep
238+
python tools/boostdep/depinst/depinst.py --git_args "--jobs $GIT_FETCH_JOBS" $LIBRARY
239+
- name: Configure and build benchmarks
240+
run: |
241+
cd ../boost-root
242+
cmake -S . -B __build__ -A ${{ matrix.cmake_arch }} \
243+
-DBUILD_TESTING=OFF \
244+
-DBOOST_INCLUDE_LIBRARIES="$LIBRARY;safe_numerics;random" \
245+
-DBOOST_SAFE_NUMBERS_BUILD_BENCHMARKS=ON
246+
cmake --build __build__ --config Release --parallel 4 \
247+
--target benchmark_unsigned_operations benchmark_signed_operations benchmark_float_operations
248+
- name: Run benchmarks and render results
249+
run: |
250+
cd ../boost-root
251+
OUT="$GITHUB_WORKSPACE/bench-results"
252+
mkdir -p "$OUT"
253+
find_exe() { find __build__ -type f \( -name "$1" -o -name "$1.exe" \) | head -1; }
254+
"$(find_exe benchmark_unsigned_operations)" $BENCH_ARGS --benchmark_out="$OUT/unsigned.json"
255+
"$(find_exe benchmark_signed_operations)" $BENCH_ARGS --benchmark_out="$OUT/signed.json"
256+
"$(find_exe benchmark_float_operations)" $BENCH_ARGS --benchmark_out="$OUT/float.json"
257+
python "$GITHUB_WORKSPACE/test/benchmarks/render_results.py" \
258+
--title "${{ matrix.title }}" --anchor "${{ matrix.anchor }}" --desc "${{ matrix.desc }}" \
259+
--unsigned "$OUT/unsigned.json" --signed "$OUT/signed.json" --float "$OUT/float.json" \
260+
> "$OUT/section.adoc"
261+
echo "==================== ${{ matrix.title }} ===================="
262+
cat "$OUT/section.adoc"
263+
- name: Upload results
264+
if: always()
265+
uses: actions/upload-artifact@v6
266+
with:
267+
name: benchmarks-windows-${{ matrix.arch }}
268+
path: bench-results
269+
if-no-files-found: warn

‎CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,13 @@ if(BUILD_TESTING AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/CMakeLists.txt")
2626
add_subdirectory(test)
2727

2828
endif()
29+
30+
# Opt-in benchmarks. Fetches Google Benchmark via FetchContent so the library
31+
# stays self-contained and the benchmarks build the same way locally and in CI.
32+
option(BOOST_SAFE_NUMBERS_BUILD_BENCHMARKS "Build the Boost.SafeNumbers benchmarks (fetches Google Benchmark)" OFF)
33+
34+
if(BOOST_SAFE_NUMBERS_BUILD_BENCHMARKS)
35+
36+
add_subdirectory(test/benchmarks)
37+
38+
endif()

0 commit comments

Comments
 (0)