Skip to content

Commit b628ed8

Browse files
vvishvvish
authored
Added continious benchmarking for algorithms
2 parents 4c47d3f + 1e56bcb commit b628ed8

6 files changed

Lines changed: 382 additions & 2 deletions

File tree

‎.github/workflows/benchmark-pr.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
name: Benchmark-PR (build)
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
7+
permissions:
8+
contents: read
9+
10+
concurrency:
11+
group: benchmark-pr-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
bench-linux:
16+
name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}'
17+
runs-on: ubuntu-latest
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
sys:
22+
- { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' }
23+
- { compiler: 'gcc', version: '14', flags: 'avx' }
24+
- { compiler: 'gcc', version: '13', flags: 'avx512' }
25+
- { compiler: 'gcc', version: '13', flags: 'avx512pf' }
26+
- { compiler: 'gcc', version: '13', flags: 'avx512vbmi' }
27+
- { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' }
28+
- { compiler: 'gcc', version: '13', flags: 'avx512vnni' }
29+
- { compiler: 'clang', version: '17', flags: 'sse3' }
30+
- { compiler: 'clang', version: '17', flags: 'avx' }
31+
32+
defaults:
33+
run:
34+
shell: bash -l {0}
35+
36+
steps:
37+
- name: Setup GCC compiler
38+
if: ${{ matrix.sys.compiler == 'gcc' }}
39+
run: |
40+
GCC_VERSION=${{ matrix.sys.version }}
41+
sudo apt-get update
42+
sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION
43+
echo "CC=gcc-$GCC_VERSION" >> $GITHUB_ENV
44+
echo "CXX=g++-$GCC_VERSION" >> $GITHUB_ENV
45+
46+
- name: Setup Clang compiler
47+
if: ${{ matrix.sys.compiler == 'clang' }}
48+
run: |
49+
LLVM_VERSION=${{ matrix.sys.version }}
50+
sudo apt-get update
51+
sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++
52+
sudo ln -sf /usr/include/asm-generic /usr/include/asm
53+
echo "CC=clang-$LLVM_VERSION" >> $GITHUB_ENV
54+
echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV
55+
56+
- uses: actions/checkout@v4
57+
58+
- name: Set conda environment
59+
uses: mamba-org/setup-micromamba@main
60+
with:
61+
environment-name: myenv
62+
environment-file: environment-dev.yml
63+
init-shell: bash
64+
cache-downloads: true
65+
66+
- name: Setup SDE
67+
if: startsWith(matrix.sys.flags, 'avx512')
68+
run: sh install_sde.sh
69+
70+
- name: Configure CMake
71+
env:
72+
CC: ${{ env.CC }}
73+
CXX: ${{ env.CXX }}
74+
run: |
75+
CMAKE_EXTRA_ARGS=""
76+
if [[ '${{ matrix.sys.flags }}' == 'avx' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=sandybridge"; fi
77+
if [[ '${{ matrix.sys.flags }}' == 'sse3' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=nocona"; fi
78+
if [[ '${{ matrix.sys.flags }}' == 'avx512' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=skylake-avx512"; fi
79+
if [[ '${{ matrix.sys.flags }}' == 'avx512pf' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knl"; fi
80+
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=cannonlake"; fi
81+
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=icelake-server"; fi
82+
if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knm"; fi
83+
cmake -S . -B _bench_build \
84+
-G Ninja \
85+
-DCMAKE_BUILD_TYPE=Release \
86+
-DCMAKE_C_COMPILER=$CC \
87+
-DCMAKE_CXX_COMPILER=$CXX \
88+
$CMAKE_EXTRA_ARGS \
89+
-DBUILD_BENCHMARKS=ON
90+
91+
- name: Build
92+
run: ninja -C _bench_build bench_algorithms
93+
94+
- name: Run benchmarks
95+
run: |
96+
if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then
97+
./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \
98+
./_bench_build/benchmark/bench_algorithms \
99+
--benchmark_format=json --benchmark_out=benchmark-result.json
100+
else
101+
./_bench_build/benchmark/bench_algorithms \
102+
--benchmark_format=json --benchmark_out=benchmark-result.json
103+
fi
104+
105+
- name: Upload benchmark artifact
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: bench-linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}
109+
path: benchmark-result.json
110+
if-no-files-found: error
111+
112+
bench-macos:
113+
name: 'macos-${{ matrix.os }}'
114+
runs-on: macos-${{ matrix.os }}
115+
strategy:
116+
fail-fast: false
117+
matrix:
118+
os: [14, 15]
119+
120+
defaults:
121+
run:
122+
shell: bash -e -l {0}
123+
124+
steps:
125+
- uses: actions/checkout@v4
126+
127+
- name: Set conda environment
128+
uses: mamba-org/setup-micromamba@main
129+
with:
130+
environment-name: myenv
131+
environment-file: environment-dev.yml
132+
init-shell: bash
133+
cache-downloads: true
134+
135+
- name: Configure CMake
136+
run: |
137+
cmake -S . -B _bench_build \
138+
-DCMAKE_BUILD_TYPE=Release \
139+
-DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
140+
-DBUILD_BENCHMARKS=ON
141+
142+
- name: Build
143+
run: cmake --build _bench_build --target bench_algorithms --parallel 8
144+
145+
- name: Run benchmarks
146+
run: |
147+
./_bench_build/benchmark/bench_algorithms \
148+
--benchmark_format=json --benchmark_out=benchmark-result.json
149+
150+
- name: Upload benchmark artifact
151+
uses: actions/upload-artifact@v4
152+
with:
153+
name: bench-macos-${{ matrix.os }}
154+
path: benchmark-result.json
155+
if-no-files-found: error

‎CMakeLists.txt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,18 @@ target_include_directories(xsimd-algorithm INTERFACE
3636
target_compile_features(xsimd-algorithm INTERFACE cxx_std_20)
3737
target_link_libraries(xsimd-algorithm INTERFACE xsimd)
3838

39-
OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF)
39+
OPTION(BUILD_TESTS "Build xsimd-algorithm test suite" OFF)
40+
OPTION(BUILD_BENCHMARKS "Build xsimd-algorithm benchmarks" OFF)
4041

4142
if(BUILD_TESTS)
4243
enable_testing()
4344
add_subdirectory(test)
4445
endif()
4546

47+
if(BUILD_BENCHMARKS)
48+
add_subdirectory(benchmark)
49+
endif()
50+
4651
# Installation
4752
# ============
4853

‎benchmark/CMakeLists.txt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
############################################################################
2+
# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and #
3+
# Martin Renou #
4+
# Copyright (c) QuantStack #
5+
# Copyright (c) Serge Guelton #
6+
# #
7+
# Distributed under the terms of the BSD 3-Clause License. #
8+
# #
9+
# The full license is in the file LICENSE, distributed with this software. #
10+
############################################################################
11+
12+
cmake_minimum_required(VERSION 3.8)
13+
14+
project(xsimd-algorithm-benchmark)
15+
16+
if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
17+
find_package(xsimd-algorithm REQUIRED CONFIG)
18+
endif ()
19+
20+
find_package(benchmark REQUIRED)
21+
22+
add_executable(bench_algorithms bench_algorithms.cpp)
23+
24+
target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark)
25+
26+
target_compile_options(bench_algorithms PRIVATE
27+
$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-O3>)
28+
29+
set_target_properties(bench_algorithms PROPERTIES
30+
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark)

‎benchmark/bench_algorithms.cpp‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
// bench_algorithms.cpp – xsimd algorithms vs std equivalents
2+
//
3+
// Build: cmake -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release ...
4+
// Run: ./bench_algorithms --benchmark_format=json --benchmark_out=result.json
5+
// (add --benchmark_min_time=2x for more stable numbers on fast machines)
6+
7+
#include <algorithm>
8+
#include <numeric>
9+
#include <vector>
10+
11+
#include <benchmark/benchmark.h>
12+
13+
#include "xsimd_algorithm/stl/arange.hpp"
14+
#include "xsimd_algorithm/stl/reduce.hpp"
15+
#include "xsimd_algorithm/stl/transform.hpp"
16+
17+
template <typename T>
18+
static std::vector<T> make_input(std::size_t n)
19+
{
20+
std::vector<T> v(n);
21+
std::iota(v.begin(), v.end(), T(1));
22+
return v;
23+
}
24+
25+
template <typename T>
26+
static void BM_std_iota(benchmark::State& state)
27+
{
28+
std::vector<T> v(static_cast<std::size_t>(state.range(0)));
29+
for (auto _ : state)
30+
{
31+
std::iota(v.begin(), v.end(), T(0));
32+
benchmark::DoNotOptimize(v.data());
33+
benchmark::ClobberMemory();
34+
}
35+
state.SetItemsProcessed(state.iterations() * state.range(0));
36+
}
37+
38+
template <typename T>
39+
static void BM_xsimd_arange(benchmark::State& state)
40+
{
41+
std::vector<T> v(static_cast<std::size_t>(state.range(0)));
42+
for (auto _ : state)
43+
{
44+
xsimd::arange(v.begin(), v.end(), T(0), T(1));
45+
benchmark::DoNotOptimize(v.data());
46+
benchmark::ClobberMemory();
47+
}
48+
state.SetItemsProcessed(state.iterations() * state.range(0));
49+
}
50+
51+
template <typename T>
52+
static void BM_std_transform_unary(benchmark::State& state)
53+
{
54+
auto src = make_input<T>(static_cast<std::size_t>(state.range(0)));
55+
std::vector<T> dst(src.size());
56+
for (auto _ : state)
57+
{
58+
std::transform(src.begin(), src.end(), dst.begin(),
59+
[](T x) noexcept
60+
{ return x * x; });
61+
benchmark::DoNotOptimize(dst.data());
62+
benchmark::ClobberMemory();
63+
}
64+
state.SetItemsProcessed(state.iterations() * state.range(0));
65+
}
66+
67+
template <typename T>
68+
static void BM_xsimd_transform_unary(benchmark::State& state)
69+
{
70+
auto src = make_input<T>(static_cast<std::size_t>(state.range(0)));
71+
std::vector<T> dst(src.size());
72+
for (auto _ : state)
73+
{
74+
xsimd::transform(src.begin(), src.end(), dst.begin(),
75+
[](auto x) noexcept
76+
{ return x * x; });
77+
benchmark::DoNotOptimize(dst.data());
78+
benchmark::ClobberMemory();
79+
}
80+
state.SetItemsProcessed(state.iterations() * state.range(0));
81+
}
82+
83+
template <typename T>
84+
static void BM_std_transform_binary(benchmark::State& state)
85+
{
86+
std::size_t n = static_cast<std::size_t>(state.range(0));
87+
auto a = make_input<T>(n);
88+
auto b = make_input<T>(n);
89+
std::vector<T> dst(n);
90+
for (auto _ : state)
91+
{
92+
std::transform(a.begin(), a.end(), b.begin(), dst.begin(),
93+
[](T x, T y) noexcept
94+
{ return x + y; });
95+
benchmark::DoNotOptimize(dst.data());
96+
benchmark::ClobberMemory();
97+
}
98+
state.SetItemsProcessed(state.iterations() * state.range(0));
99+
}
100+
101+
template <typename T>
102+
static void BM_xsimd_transform_binary(benchmark::State& state)
103+
{
104+
std::size_t n = static_cast<std::size_t>(state.range(0));
105+
auto a = make_input<T>(n);
106+
auto b = make_input<T>(n);
107+
std::vector<T> dst(n);
108+
for (auto _ : state)
109+
{
110+
xsimd::transform(a.begin(), a.end(), b.begin(), dst.begin(),
111+
[](auto x, auto y) noexcept
112+
{ return x + y; });
113+
benchmark::DoNotOptimize(dst.data());
114+
benchmark::ClobberMemory();
115+
}
116+
state.SetItemsProcessed(state.iterations() * state.range(0));
117+
}
118+
119+
template <typename T>
120+
static void BM_std_reduce(benchmark::State& state)
121+
{
122+
auto v = make_input<T>(static_cast<std::size_t>(state.range(0)));
123+
for (auto _ : state)
124+
{
125+
T result = std::reduce(v.begin(), v.end(), T(0));
126+
benchmark::DoNotOptimize(result);
127+
}
128+
state.SetItemsProcessed(state.iterations() * state.range(0));
129+
}
130+
131+
template <typename T>
132+
static void BM_xsimd_reduce(benchmark::State& state)
133+
{
134+
auto v = make_input<T>(static_cast<std::size_t>(state.range(0)));
135+
for (auto _ : state)
136+
{
137+
T result = xsimd::reduce(v.begin(), v.end(), T(0));
138+
benchmark::DoNotOptimize(result);
139+
}
140+
state.SetItemsProcessed(state.iterations() * state.range(0));
141+
}
142+
143+
static constexpr long long kSmall = 1LL << 12;
144+
static constexpr long long kMedium = 1LL << 18;
145+
static constexpr long long kLarge = 1LL << 22;
146+
147+
#define BENCH_ARGS ->Arg(kSmall)->Arg(kMedium)->Arg(kLarge)
148+
149+
// arange / iota
150+
BENCHMARK_TEMPLATE(BM_std_iota, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
151+
BENCHMARK_TEMPLATE(BM_xsimd_arange, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
152+
BENCHMARK_TEMPLATE(BM_std_iota, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
153+
BENCHMARK_TEMPLATE(BM_xsimd_arange, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
154+
155+
// transform unary
156+
BENCHMARK_TEMPLATE(BM_std_transform_unary, float)
157+
BENCH_ARGS;
158+
BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, float)
159+
BENCH_ARGS;
160+
BENCHMARK_TEMPLATE(BM_std_transform_unary, double)
161+
BENCH_ARGS;
162+
BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, double)
163+
BENCH_ARGS;
164+
165+
// transform binary
166+
BENCHMARK_TEMPLATE(BM_std_transform_binary, float)
167+
BENCH_ARGS;
168+
BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, float)
169+
BENCH_ARGS;
170+
BENCHMARK_TEMPLATE(BM_std_transform_binary, double)
171+
BENCH_ARGS;
172+
BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, double)
173+
BENCH_ARGS;
174+
175+
// reduce
176+
BENCHMARK_TEMPLATE(BM_std_reduce, float)
177+
BENCH_ARGS;
178+
BENCHMARK_TEMPLATE(BM_xsimd_reduce, float)
179+
BENCH_ARGS;
180+
BENCHMARK_TEMPLATE(BM_std_reduce, double)
181+
BENCH_ARGS;
182+
BENCHMARK_TEMPLATE(BM_xsimd_reduce, double)
183+
BENCH_ARGS;
184+
185+
BENCHMARK_MAIN();

0 commit comments

Comments
 (0)