From 1e56bcb68a969db76b8255f332209dea21078e88 Mon Sep 17 00:00:00 2001 From: vvish Date: Sun, 12 Apr 2026 19:05:06 +0200 Subject: [PATCH] Benchmark (tests) --- .github/workflows/benchmark-pr.yml | 155 ++++++++++++++++++++++++ CMakeLists.txt | 7 +- benchmark/CMakeLists.txt | 30 +++++ benchmark/bench_algorithms.cpp | 185 +++++++++++++++++++++++++++++ environment-dev.yml | 3 +- test/CMakeLists.txt | 4 + 6 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark-pr.yml create mode 100644 benchmark/CMakeLists.txt create mode 100644 benchmark/bench_algorithms.cpp diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml new file mode 100644 index 0000000..219ad87 --- /dev/null +++ b/.github/workflows/benchmark-pr.yml @@ -0,0 +1,155 @@ +name: Benchmark-PR (build) + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: benchmark-pr-${{ github.ref }} + cancel-in-progress: true + +jobs: + bench-linux: + name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sys: + - { compiler: 'gcc', version: '12', flags: 'force_no_instr_set' } + - { compiler: 'gcc', version: '14', flags: 'avx' } + - { compiler: 'gcc', version: '13', flags: 'avx512' } + - { compiler: 'gcc', version: '13', flags: 'avx512pf' } + - { compiler: 'gcc', version: '13', flags: 'avx512vbmi' } + - { compiler: 'gcc', version: '14', flags: 'avx512vbmi2' } + - { compiler: 'gcc', version: '13', flags: 'avx512vnni' } + - { compiler: 'clang', version: '17', flags: 'sse3' } + - { compiler: 'clang', version: '17', flags: 'avx' } + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Setup GCC compiler + if: ${{ matrix.sys.compiler == 'gcc' }} + run: | + GCC_VERSION=${{ matrix.sys.version }} + sudo apt-get update + sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION + echo "CC=gcc-$GCC_VERSION" >> $GITHUB_ENV + echo "CXX=g++-$GCC_VERSION" >> $GITHUB_ENV + + - name: Setup Clang compiler + if: ${{ matrix.sys.compiler == 'clang' }} + run: | + LLVM_VERSION=${{ matrix.sys.version }} + sudo apt-get update + sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++ + sudo ln -sf /usr/include/asm-generic /usr/include/asm + echo "CC=clang-$LLVM_VERSION" >> $GITHUB_ENV + echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@main + with: + environment-name: myenv + environment-file: environment-dev.yml + init-shell: bash + cache-downloads: true + + - name: Setup SDE + if: startsWith(matrix.sys.flags, 'avx512') + run: sh install_sde.sh + + - name: Configure CMake + env: + CC: ${{ env.CC }} + CXX: ${{ env.CXX }} + run: | + CMAKE_EXTRA_ARGS="" + if [[ '${{ matrix.sys.flags }}' == 'avx' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=sandybridge"; fi + if [[ '${{ matrix.sys.flags }}' == 'sse3' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=nocona"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=skylake-avx512"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512pf' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knl"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=cannonlake"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=icelake-server"; fi + if [[ '${{ matrix.sys.flags }}' == 'avx512vnni' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knm"; fi + cmake -S . -B _bench_build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=$CC \ + -DCMAKE_CXX_COMPILER=$CXX \ + $CMAKE_EXTRA_ARGS \ + -DBUILD_BENCHMARKS=ON + + - name: Build + run: ninja -C _bench_build bench_algorithms + + - name: Run benchmarks + run: | + if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then + ./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \ + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_format=json --benchmark_out=benchmark-result.json + else + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_format=json --benchmark_out=benchmark-result.json + fi + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bench-linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }} + path: benchmark-result.json + if-no-files-found: error + + bench-macos: + name: 'macos-${{ matrix.os }}' + runs-on: macos-${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [14, 15] + + defaults: + run: + shell: bash -e -l {0} + + steps: + - uses: actions/checkout@v4 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@main + with: + environment-name: myenv + environment-file: environment-dev.yml + init-shell: bash + cache-downloads: true + + - name: Configure CMake + run: | + cmake -S . -B _bench_build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + -DBUILD_BENCHMARKS=ON + + - name: Build + run: cmake --build _bench_build --target bench_algorithms --parallel 8 + + - name: Run benchmarks + run: | + ./_bench_build/benchmark/bench_algorithms \ + --benchmark_format=json --benchmark_out=benchmark-result.json + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: bench-macos-${{ matrix.os }} + path: benchmark-result.json + if-no-files-found: error diff --git a/CMakeLists.txt b/CMakeLists.txt index 85b4606..c22154a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,13 +36,18 @@ target_include_directories(xsimd-algorithm INTERFACE target_compile_features(xsimd-algorithm INTERFACE cxx_std_20) target_link_libraries(xsimd-algorithm INTERFACE xsimd) -OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF) +OPTION(BUILD_TESTS "Build xsimd-algorithm test suite" OFF) +OPTION(BUILD_BENCHMARKS "Build xsimd-algorithm benchmarks" OFF) if(BUILD_TESTS) enable_testing() add_subdirectory(test) endif() +if(BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() + # Installation # ============ diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 0000000..ff3c97f --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,30 @@ +############################################################################ +# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and # +# Martin Renou # +# Copyright (c) QuantStack # +# Copyright (c) Serge Guelton # +# # +# Distributed under the terms of the BSD 3-Clause License. # +# # +# The full license is in the file LICENSE, distributed with this software. # +############################################################################ + +cmake_minimum_required(VERSION 3.8) + +project(xsimd-algorithm-benchmark) + +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + find_package(xsimd-algorithm REQUIRED CONFIG) +endif () + +find_package(benchmark REQUIRED) + +add_executable(bench_algorithms bench_algorithms.cpp) + +target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark) + +target_compile_options(bench_algorithms PRIVATE + $<$:-O3>) + +set_target_properties(bench_algorithms PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark) diff --git a/benchmark/bench_algorithms.cpp b/benchmark/bench_algorithms.cpp new file mode 100644 index 0000000..58267cd --- /dev/null +++ b/benchmark/bench_algorithms.cpp @@ -0,0 +1,185 @@ +// bench_algorithms.cpp – xsimd algorithms vs std equivalents +// +// Build: cmake -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release ... +// Run: ./bench_algorithms --benchmark_format=json --benchmark_out=result.json +// (add --benchmark_min_time=2x for more stable numbers on fast machines) + +#include +#include +#include + +#include + +#include "xsimd_algorithm/stl/arange.hpp" +#include "xsimd_algorithm/stl/reduce.hpp" +#include "xsimd_algorithm/stl/transform.hpp" + +template +static std::vector make_input(std::size_t n) +{ + std::vector v(n); + std::iota(v.begin(), v.end(), T(1)); + return v; +} + +template +static void BM_std_iota(benchmark::State& state) +{ + std::vector v(static_cast(state.range(0))); + for (auto _ : state) + { + std::iota(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_xsimd_arange(benchmark::State& state) +{ + std::vector v(static_cast(state.range(0))); + for (auto _ : state) + { + xsimd::arange(v.begin(), v.end(), T(0), T(1)); + benchmark::DoNotOptimize(v.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_std_transform_unary(benchmark::State& state) +{ + auto src = make_input(static_cast(state.range(0))); + std::vector dst(src.size()); + for (auto _ : state) + { + std::transform(src.begin(), src.end(), dst.begin(), + [](T x) noexcept + { return x * x; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_xsimd_transform_unary(benchmark::State& state) +{ + auto src = make_input(static_cast(state.range(0))); + std::vector dst(src.size()); + for (auto _ : state) + { + xsimd::transform(src.begin(), src.end(), dst.begin(), + [](auto x) noexcept + { return x * x; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_std_transform_binary(benchmark::State& state) +{ + std::size_t n = static_cast(state.range(0)); + auto a = make_input(n); + auto b = make_input(n); + std::vector dst(n); + for (auto _ : state) + { + std::transform(a.begin(), a.end(), b.begin(), dst.begin(), + [](T x, T y) noexcept + { return x + y; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_xsimd_transform_binary(benchmark::State& state) +{ + std::size_t n = static_cast(state.range(0)); + auto a = make_input(n); + auto b = make_input(n); + std::vector dst(n); + for (auto _ : state) + { + xsimd::transform(a.begin(), a.end(), b.begin(), dst.begin(), + [](auto x, auto y) noexcept + { return x + y; }); + benchmark::DoNotOptimize(dst.data()); + benchmark::ClobberMemory(); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_std_reduce(benchmark::State& state) +{ + auto v = make_input(static_cast(state.range(0))); + for (auto _ : state) + { + T result = std::reduce(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +template +static void BM_xsimd_reduce(benchmark::State& state) +{ + auto v = make_input(static_cast(state.range(0))); + for (auto _ : state) + { + T result = xsimd::reduce(v.begin(), v.end(), T(0)); + benchmark::DoNotOptimize(result); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} + +static constexpr long long kSmall = 1LL << 12; +static constexpr long long kMedium = 1LL << 18; +static constexpr long long kLarge = 1LL << 22; + +#define BENCH_ARGS ->Arg(kSmall)->Arg(kMedium)->Arg(kLarge) + +// arange / iota +BENCHMARK_TEMPLATE(BM_std_iota, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge); +BENCHMARK_TEMPLATE(BM_xsimd_arange, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge); +BENCHMARK_TEMPLATE(BM_std_iota, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge); +BENCHMARK_TEMPLATE(BM_xsimd_arange, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge); + +// transform unary +BENCHMARK_TEMPLATE(BM_std_transform_unary, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_std_transform_unary, double) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, double) +BENCH_ARGS; + +// transform binary +BENCHMARK_TEMPLATE(BM_std_transform_binary, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_std_transform_binary, double) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, double) +BENCH_ARGS; + +// reduce +BENCHMARK_TEMPLATE(BM_std_reduce, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_reduce, float) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_std_reduce, double) +BENCH_ARGS; +BENCHMARK_TEMPLATE(BM_xsimd_reduce, double) +BENCH_ARGS; + +BENCHMARK_MAIN(); diff --git a/environment-dev.yml b/environment-dev.yml index cb1648e..2690b4a 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,4 +5,5 @@ dependencies: - cmake - xsimd=13.2.0 - doctest -- ninja \ No newline at end of file +- ninja +- benchmark \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2285b23..bb3a5b9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,8 @@ if(CMAKE_CXX_COMPILER_ID MATCHES Clang AND MSVC AND WIN32) # We are using clang- set(CMAKE_EXE_LINKER_FLAGS /MANIFEST:NO) endif() +if(BUILD_TESTS OR CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(XSIMD_ALGORITHM_TESTS main.cpp test_arange.cpp @@ -96,3 +98,5 @@ endif() if (XSIMD_ENABLE_WERROR) target_compile_options(test_xsimd_algorithm PRIVATE -Werror -Wall -DXSIMD_SKIP_ON_WERROR) endif() + +endif() # BUILD_TESTS OR standalone