Skip to content

Commit 14f9fe1

Browse files
vvishvvish
committed
Added continious benchmarking for algorithms
1 parent 07fbbd5 commit 14f9fe1

7 files changed

Lines changed: 469 additions & 2 deletions

File tree

‎.github/workflows/benchmark-pr-comment.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
name: Benchmark – PR (comment)
2+
3+
on:
4+
workflow_run:
5+
workflows: ["Benchmark – PR (build)"]
6+
types: [completed]
7+
8+
permissions:
9+
contents: read
10+
actions: read
11+
pull-requests: write
12+
13+
jobs:
14+
comment-linux:
15+
if: >
16+
github.event.workflow_run.event == 'pull_request' &&
17+
github.event.workflow_run.conclusion == 'success'
18+
name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}'
19+
runs-on: ubuntu-latest
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
sys:
24+
- { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' }
25+
- { compiler: 'gcc', version: '14', flags: 'avx' }
26+
- { compiler: 'gcc', version: '13', flags: 'avx512' }
27+
- { compiler: 'gcc', version: '13', flags: 'avx512pf' }
28+
- { compiler: 'gcc', version: '13', flags: 'avx512vbmi' }
29+
- { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' }
30+
- { compiler: 'gcc', version: '13', flags: 'avx512vnni' }
31+
- { compiler: 'clang', version: '17', flags: 'sse3' }
32+
- { compiler: 'clang', version: '17', flags: 'avx' }
33+
34+
steps:
35+
- name: Download benchmark artifact
36+
uses: actions/download-artifact@v4
37+
with:
38+
name: bench-linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}-${{ github.event.workflow_run.head_sha }}
39+
run-id: ${{ github.event.workflow_run.id }}
40+
github-token: ${{ secrets.GITHUB_TOKEN }}
41+
42+
- name: Compare benchmark results
43+
uses: benchmark-action/github-action-benchmark@v1
44+
with:
45+
tool: googlecpp
46+
output-file-path: benchmark-result.json
47+
github-token: ${{ secrets.GITHUB_TOKEN }}
48+
auto-push: false
49+
gh-pages-branch: gh-pages
50+
benchmark-data-dir-path: dev/bench/linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}
51+
comment-on-alert: true
52+
alert-threshold: '150%'
53+
fail-on-alert: false
54+
comment-always: true
55+
56+
comment-macos:
57+
if: >
58+
github.event.workflow_run.event == 'pull_request' &&
59+
github.event.workflow_run.conclusion == 'success'
60+
name: 'macos-${{ matrix.os }}'
61+
runs-on: ubuntu-latest
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
os: [14, 15]
66+
67+
steps:
68+
- name: Download benchmark artifact
69+
uses: actions/download-artifact@v4
70+
with:
71+
name: bench-macos-${{ matrix.os }}-${{ github.event.workflow_run.head_sha }}
72+
run-id: ${{ github.event.workflow_run.id }}
73+
github-token: ${{ secrets.GITHUB_TOKEN }}
74+
75+
- name: Compare benchmark results
76+
uses: benchmark-action/github-action-benchmark@v1
77+
with:
78+
tool: googlecpp
79+
output-file-path: benchmark-result.json
80+
github-token: ${{ secrets.GITHUB_TOKEN }}
81+
auto-push: false
82+
gh-pages-branch: gh-pages
83+
benchmark-data-dir-path: dev/bench/macos-${{ matrix.os }}
84+
comment-on-alert: true
85+
alert-threshold: '150%'
86+
fail-on-alert: false
87+
comment-always: true

‎.github/workflows/benchmark-pr.yml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
name: Benchmark – PR (build)
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
7+
permissions:
8+
contents: read
9+
10+
concurrency:
11+
group: benchmark-pr-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
bench-linux:
16+
name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}'
17+
runs-on: ubuntu-latest
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
sys:
22+
- { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' }
23+
- { compiler: 'gcc', version: '14', flags: 'avx' }
24+
- { compiler: 'gcc', version: '13', flags: 'avx512' }
25+
- { compiler: 'gcc', version: '13', flags: 'avx512pf' }
26+
- { compiler: 'gcc', version: '13', flags: 'avx512vbmi' }
27+
- { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' }
28+
- { compiler: 'gcc', version: '13', flags: 'avx512vnni' }
29+
- { compiler: 'clang', version: '17', flags: 'sse3' }
30+
- { compiler: 'clang', version: '17', flags: 'avx' }
31+
32+
defaults:
33+
run:
34+
shell: bash -l {0}
35+
36+
steps:
37+
- name: Setup GCC compiler
38+
if: ${{ matrix.sys.compiler == 'gcc' }}
39+
run: |
40+
GCC_VERSION=${{ matrix.sys.version }}
41+
sudo apt-get update
42+
sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION
43+
echo "CC=gcc-$GCC_VERSION" >> $GITHUB_ENV
44+
echo "CXX=g++-$GCC_VERSION" >> $GITHUB_ENV
45+
46+
- name: Setup Clang compiler
47+
if: ${{ matrix.sys.compiler == 'clang' }}
48+
run: |
49+
LLVM_VERSION=${{ matrix.sys.version }}
50+
sudo apt-get update
51+
sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++
52+
sudo ln -sf /usr/include/asm-generic /usr/include/asm
53+
echo "CC=clang-$LLVM_VERSION" >> $GITHUB_ENV
54+
echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV
55+
56+
- uses: actions/checkout@v4
57+
58+
- name: Set conda environment
59+
uses: mamba-org/setup-micromamba@main
60+
with:
61+
environment-name: myenv
62+
environment-file: environment-dev.yml
63+
init-shell: bash
64+
cache-downloads: true
65+
66+
- name: Setup SDE
67+
if: startsWith(matrix.sys.flags, 'avx512')
68+
run: sh install_sde.sh
69+
70+
- name: Configure CMake
71+
env:
72+
CC: ${{ env.CC }}
73+
CXX: ${{ env.CXX }}
74+
run: |
75+
CMAKE_EXTRA_ARGS=""
76+
if [[ '${{ matrix.sys.flags }}' == 'avx' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=sandybridge"; fi
77+
if [[ '${{ matrix.sys.flags }}' == 'sse3' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=nocona"; fi
78+
if [[ '${{ matrix.sys.flags }}' == 'avx512' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=skylake-avx512"; fi
79+
if [[ '${{ matrix.sys.flags }}' == 'avx512pf' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knl"; fi
80+
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=cannonlake"; fi
81+
if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=icelake-server"; fi
82+
if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knm"; fi
83+
cmake -S . -B _bench_build \
84+
-G Ninja \
85+
-DCMAKE_BUILD_TYPE=Release \
86+
-DCMAKE_C_COMPILER=$CC \
87+
-DCMAKE_CXX_COMPILER=$CXX \
88+
$CMAKE_EXTRA_ARGS \
89+
-DBUILD_BENCHMARKS=ON
90+
91+
- name: Build
92+
run: ninja -C _bench_build bench_algorithms
93+
94+
- name: Run benchmarks
95+
run: |
96+
if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then
97+
./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \
98+
./_bench_build/benchmark/bench_algorithms \
99+
--benchmark_format=json --benchmark_out=benchmark-result.json
100+
else
101+
./_bench_build/benchmark/bench_algorithms \
102+
--benchmark_format=json --benchmark_out=benchmark-result.json
103+
fi
104+
105+
- name: Upload benchmark artifact
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: bench-linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}-${{ github.sha }}
109+
path: benchmark-result.json
110+
if-no-files-found: error
111+
112+
bench-macos:
113+
name: 'macos-${{ matrix.os }}'
114+
runs-on: macos-${{ matrix.os }}
115+
strategy:
116+
fail-fast: false
117+
matrix:
118+
os: [14, 15]
119+
120+
defaults:
121+
run:
122+
shell: bash -e -l {0}
123+
124+
steps:
125+
- uses: actions/checkout@v4
126+
127+
- name: Set conda environment
128+
uses: mamba-org/setup-micromamba@main
129+
with:
130+
environment-name: myenv
131+
environment-file: environment-dev.yml
132+
init-shell: bash
133+
cache-downloads: true
134+
135+
- name: Configure CMake
136+
run: |
137+
cmake -S . -B _bench_build \
138+
-DCMAKE_BUILD_TYPE=Release \
139+
-DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
140+
-DBUILD_BENCHMARKS=ON
141+
142+
- name: Build
143+
run: cmake --build _bench_build --target bench_algorithms --parallel 8
144+
145+
- name: Run benchmarks
146+
run: |
147+
./_bench_build/benchmark/bench_algorithms \
148+
--benchmark_format=json --benchmark_out=benchmark-result.json
149+
150+
- name: Upload benchmark artifact
151+
uses: actions/upload-artifact@v4
152+
with:
153+
name: bench-macos-${{ matrix.os }}-${{ github.sha }}
154+
path: benchmark-result.json
155+
if-no-files-found: error

‎CMakeLists.txt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,18 @@ target_include_directories(xsimd-algorithm INTERFACE
3636
target_compile_features(xsimd-algorithm INTERFACE cxx_std_20)
3737
target_link_libraries(xsimd-algorithm INTERFACE xsimd)
3838

39-
OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF)
39+
OPTION(BUILD_TESTS "Build xsimd-algorithm test suite" OFF)
40+
OPTION(BUILD_BENCHMARKS "Build xsimd-algorithm benchmarks" OFF)
4041

4142
if(BUILD_TESTS)
4243
enable_testing()
4344
add_subdirectory(test)
4445
endif()
4546

47+
if(BUILD_BENCHMARKS)
48+
add_subdirectory(benchmark)
49+
endif()
50+
4651
# Installation
4752
# ============
4853

‎benchmark/CMakeLists.txt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
############################################################################
2+
# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and #
3+
# Martin Renou #
4+
# Copyright (c) QuantStack #
5+
# Copyright (c) Serge Guelton #
6+
# #
7+
# Distributed under the terms of the BSD 3-Clause License. #
8+
# #
9+
# The full license is in the file LICENSE, distributed with this software. #
10+
############################################################################
11+
12+
cmake_minimum_required(VERSION 3.8)
13+
14+
project(xsimd-algorithm-benchmark)
15+
16+
if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
17+
find_package(xsimd-algorithm REQUIRED CONFIG)
18+
endif ()
19+
20+
find_package(benchmark REQUIRED)
21+
22+
add_executable(bench_algorithms bench_algorithms.cpp)
23+
24+
target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark)
25+
26+
target_compile_options(bench_algorithms PRIVATE
27+
$<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-O3>)
28+
29+
set_target_properties(bench_algorithms PROPERTIES
30+
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark)

0 commit comments

Comments
 (0)