Added continious benchmarking for algorithms

.github/workflows/benchmark-pr.yml

@@ -0,0 +1,155 @@
+name: Benchmark-PR (build)
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: benchmark-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  bench-linux:
+    name: 'linux ${{ matrix.sys.compiler }}-${{ matrix.sys.version }} ${{ matrix.sys.flags }}'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        sys:
+          - { compiler: 'gcc',   version: '12', flags: 'force_no_instr_set' }
+          - { compiler: 'gcc',   version: '14', flags: 'avx'                }
+          - { compiler: 'gcc',   version: '13', flags: 'avx512'             }
+          - { compiler: 'gcc',   version: '13', flags: 'avx512pf'           }
+          - { compiler: 'gcc',   version: '13', flags: 'avx512vbmi'         }
+          - { compiler: 'gcc',   version: '14', flags: 'avx512vbmi2'        }
+          - { compiler: 'gcc',   version: '13', flags: 'avx512vnni'         }
+          - { compiler: 'clang', version: '17', flags: 'sse3'               }
+          - { compiler: 'clang', version: '17', flags: 'avx'                }
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Setup GCC compiler
+        if: ${{ matrix.sys.compiler == 'gcc' }}
+        run: |
+          GCC_VERSION=${{ matrix.sys.version }}
+          sudo apt-get update
+          sudo apt-get --no-install-suggests --no-install-recommends install g++-$GCC_VERSION gcc-$GCC_VERSION
+          echo "CC=gcc-$GCC_VERSION"   >> $GITHUB_ENV
+          echo "CXX=g++-$GCC_VERSION"  >> $GITHUB_ENV
+
+      - name: Setup Clang compiler
+        if: ${{ matrix.sys.compiler == 'clang' }}
+        run: |
+          LLVM_VERSION=${{ matrix.sys.version }}
+          sudo apt-get update
+          sudo apt-get --no-install-suggests --no-install-recommends install clang-$LLVM_VERSION g++
+          sudo ln -sf /usr/include/asm-generic /usr/include/asm
+          echo "CC=clang-$LLVM_VERSION"    >> $GITHUB_ENV
+          echo "CXX=clang++-$LLVM_VERSION" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+
+      - name: Set conda environment
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-name: myenv
+          environment-file: environment-dev.yml
+          init-shell: bash
+          cache-downloads: true
+
+      - name: Setup SDE
+        if: startsWith(matrix.sys.flags, 'avx512')
+        run: sh install_sde.sh
+
+      - name: Configure CMake
+        env:
+          CC:  ${{ env.CC }}
+          CXX: ${{ env.CXX }}
+        run: |
+          CMAKE_EXTRA_ARGS=""
+          if [[ '${{ matrix.sys.flags }}' == 'avx'         ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=sandybridge";    fi
+          if [[ '${{ matrix.sys.flags }}' == 'sse3'        ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=nocona";         fi
+          if [[ '${{ matrix.sys.flags }}' == 'avx512'      ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=skylake-avx512"; fi
+          if [[ '${{ matrix.sys.flags }}' == 'avx512pf'    ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knl";            fi
+          if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi'  ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=cannonlake";     fi
+          if [[ '${{ matrix.sys.flags }}' == 'avx512vbmi2' ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=icelake-server"; fi
+          if [[ '${{ matrix.sys.flags }}' == 'avx512vnni'  ]]; then CMAKE_EXTRA_ARGS="-DTARGET_ARCH=knm";            fi
+          cmake -S . -B _bench_build \
+            -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER=$CC \
+            -DCMAKE_CXX_COMPILER=$CXX \
+            $CMAKE_EXTRA_ARGS \
+            -DBUILD_BENCHMARKS=ON
+
+      - name: Build
+        run: ninja -C _bench_build bench_algorithms
+
+      - name: Run benchmarks
+        run: |
+          if echo '${{ matrix.sys.flags }}' | grep -q 'avx512'; then
+            ./sde-external-9.48.0-2024-11-25-lin/sde64 -tgl -- \
+              ./_bench_build/benchmark/bench_algorithms \
+              --benchmark_format=json --benchmark_out=benchmark-result.json
+          else
+            ./_bench_build/benchmark/bench_algorithms \
+              --benchmark_format=json --benchmark_out=benchmark-result.json
+          fi
+
+      - name: Upload benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-linux-${{ matrix.sys.compiler }}-${{ matrix.sys.version }}-${{ matrix.sys.flags }}
+          path: benchmark-result.json
+          if-no-files-found: error
+
+  bench-macos:
+    name: 'macos-${{ matrix.os }}'
+    runs-on: macos-${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [14, 15]
+
+    defaults:
+      run:
+        shell: bash -e -l {0}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set conda environment
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-name: myenv
+          environment-file: environment-dev.yml
+          init-shell: bash
+          cache-downloads: true
+
+      - name: Configure CMake
+        run: |
+          cmake -S . -B _bench_build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+            -DBUILD_BENCHMARKS=ON
+
+      - name: Build
+        run: cmake --build _bench_build --target bench_algorithms --parallel 8
+
+      - name: Run benchmarks
+        run: |
+          ./_bench_build/benchmark/bench_algorithms \
+            --benchmark_format=json --benchmark_out=benchmark-result.json
+
+      - name: Upload benchmark artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-macos-${{ matrix.os }}
+          path: benchmark-result.json
+          if-no-files-found: error

CMakeLists.txt

@@ -36,13 +36,18 @@ target_include_directories(xsimd-algorithm INTERFACE
 target_compile_features(xsimd-algorithm INTERFACE cxx_std_20)
 target_link_libraries(xsimd-algorithm INTERFACE xsimd)
 
-OPTION(BUILD_TESTS "xsimd-algorithm test suite" OFF)
+OPTION(BUILD_TESTS       "Build xsimd-algorithm test suite"  OFF)
+OPTION(BUILD_BENCHMARKS  "Build xsimd-algorithm benchmarks"  OFF)
 
 if(BUILD_TESTS)
     enable_testing()
     add_subdirectory(test)
 endif()
 
+if(BUILD_BENCHMARKS)
+    add_subdirectory(benchmark)
+endif()
+
 # Installation
 # ============
 

benchmark/CMakeLists.txt

@@ -0,0 +1,30 @@
+############################################################################
+# Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         #
+# Martin Renou                                                             #
+# Copyright (c) QuantStack                                                 #
+# Copyright (c) Serge Guelton                                              #
+#                                                                          #
+# Distributed under the terms of the BSD 3-Clause License.                 #
+#                                                                          #
+# The full license is in the file LICENSE, distributed with this software. #
+############################################################################
+
+cmake_minimum_required(VERSION 3.8)
+
+project(xsimd-algorithm-benchmark)
+
+if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+    find_package(xsimd-algorithm REQUIRED CONFIG)
+endif ()
+
+find_package(benchmark REQUIRED)
+
+add_executable(bench_algorithms bench_algorithms.cpp)
+
+target_link_libraries(bench_algorithms PRIVATE xsimd-algorithm benchmark::benchmark)
+
+target_compile_options(bench_algorithms PRIVATE
+    $<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-O3>)
+
+set_target_properties(bench_algorithms PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/benchmark)

benchmark/bench_algorithms.cpp

@@ -0,0 +1,185 @@
+// bench_algorithms.cpp  –  xsimd algorithms vs std equivalents
+//
+// Build:  cmake -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release ...
+// Run:    ./bench_algorithms --benchmark_format=json --benchmark_out=result.json
+//         (add --benchmark_min_time=2x for more stable numbers on fast machines)
+
+#include <algorithm>
+#include <numeric>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+
+#include "xsimd_algorithm/stl/arange.hpp"
+#include "xsimd_algorithm/stl/reduce.hpp"
+#include "xsimd_algorithm/stl/transform.hpp"
+
+template <typename T>
+static std::vector<T> make_input(std::size_t n)
+{
+    std::vector<T> v(n);
+    std::iota(v.begin(), v.end(), T(1));
+    return v;
+}
+
+template <typename T>
+static void BM_std_iota(benchmark::State& state)
+{
+    std::vector<T> v(static_cast<std::size_t>(state.range(0)));
+    for (auto _ : state)
+    {
+        std::iota(v.begin(), v.end(), T(0));
+        benchmark::DoNotOptimize(v.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_xsimd_arange(benchmark::State& state)
+{
+    std::vector<T> v(static_cast<std::size_t>(state.range(0)));
+    for (auto _ : state)
+    {
+        xsimd::arange(v.begin(), v.end(), T(0), T(1));
+        benchmark::DoNotOptimize(v.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_std_transform_unary(benchmark::State& state)
+{
+    auto src = make_input<T>(static_cast<std::size_t>(state.range(0)));
+    std::vector<T> dst(src.size());
+    for (auto _ : state)
+    {
+        std::transform(src.begin(), src.end(), dst.begin(),
+                       [](T x) noexcept
+                       { return x * x; });
+        benchmark::DoNotOptimize(dst.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_xsimd_transform_unary(benchmark::State& state)
+{
+    auto src = make_input<T>(static_cast<std::size_t>(state.range(0)));
+    std::vector<T> dst(src.size());
+    for (auto _ : state)
+    {
+        xsimd::transform(src.begin(), src.end(), dst.begin(),
+                         [](auto x) noexcept
+                         { return x * x; });
+        benchmark::DoNotOptimize(dst.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_std_transform_binary(benchmark::State& state)
+{
+    std::size_t n = static_cast<std::size_t>(state.range(0));
+    auto a = make_input<T>(n);
+    auto b = make_input<T>(n);
+    std::vector<T> dst(n);
+    for (auto _ : state)
+    {
+        std::transform(a.begin(), a.end(), b.begin(), dst.begin(),
+                       [](T x, T y) noexcept
+                       { return x + y; });
+        benchmark::DoNotOptimize(dst.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_xsimd_transform_binary(benchmark::State& state)
+{
+    std::size_t n = static_cast<std::size_t>(state.range(0));
+    auto a = make_input<T>(n);
+    auto b = make_input<T>(n);
+    std::vector<T> dst(n);
+    for (auto _ : state)
+    {
+        xsimd::transform(a.begin(), a.end(), b.begin(), dst.begin(),
+                         [](auto x, auto y) noexcept
+                         { return x + y; });
+        benchmark::DoNotOptimize(dst.data());
+        benchmark::ClobberMemory();
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_std_reduce(benchmark::State& state)
+{
+    auto v = make_input<T>(static_cast<std::size_t>(state.range(0)));
+    for (auto _ : state)
+    {
+        T result = std::reduce(v.begin(), v.end(), T(0));
+        benchmark::DoNotOptimize(result);
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+template <typename T>
+static void BM_xsimd_reduce(benchmark::State& state)
+{
+    auto v = make_input<T>(static_cast<std::size_t>(state.range(0)));
+    for (auto _ : state)
+    {
+        T result = xsimd::reduce(v.begin(), v.end(), T(0));
+        benchmark::DoNotOptimize(result);
+    }
+    state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+
+static constexpr long long kSmall = 1LL << 12;
+static constexpr long long kMedium = 1LL << 18;
+static constexpr long long kLarge = 1LL << 22;
+
+#define BENCH_ARGS ->Arg(kSmall)->Arg(kMedium)->Arg(kLarge)
+
+// arange / iota
+BENCHMARK_TEMPLATE(BM_std_iota, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
+BENCHMARK_TEMPLATE(BM_xsimd_arange, float)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
+BENCHMARK_TEMPLATE(BM_std_iota, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
+BENCHMARK_TEMPLATE(BM_xsimd_arange, double)->Arg(kSmall)->Arg(kMedium)->Arg(kLarge);
+
+// transform unary
+BENCHMARK_TEMPLATE(BM_std_transform_unary, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_std_transform_unary, double)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_transform_unary, double)
+BENCH_ARGS;
+
+// transform binary
+BENCHMARK_TEMPLATE(BM_std_transform_binary, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_std_transform_binary, double)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_transform_binary, double)
+BENCH_ARGS;
+
+// reduce
+BENCHMARK_TEMPLATE(BM_std_reduce, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_reduce, float)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_std_reduce, double)
+BENCH_ARGS;
+BENCHMARK_TEMPLATE(BM_xsimd_reduce, double)
+BENCH_ARGS;
+
+BENCHMARK_MAIN();