Use Google Benchmark for performance tables

Author: allnes

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "3rdparty/libenvpp"]
 	path = 3rdparty/libenvpp
 	url = https://github.com/ph3at/libenvpp
+[submodule "3rdparty/benchmark"]
+	path = 3rdparty/benchmark
+	url = https://github.com/google/benchmark

3rdparty/benchmark

@@ -44,6 +44,7 @@ endforeach()
 
 message( STATUS "PPC step: Setup external projects" )
 include(cmake/gtest.cmake)
+include(cmake/benchmark.cmake)
 
 ############################## Modules ##############################
 

CMakeLists.txt

@@ -0,0 +1,44 @@
+include_guard()
+
+include(ExternalProject)
+
+ExternalProject_Add(
+  ppc_benchmark
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/benchmark"
+  PREFIX "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark"
+  BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build"
+  INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/install"
+  EXCLUDE_FROM_ALL TRUE
+  CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+             -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
+             -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
+             -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
+             -DCMAKE_CXX_STANDARD_REQUIRED=${CMAKE_CXX_STANDARD_REQUIRED}
+             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+             ${PPC_EXTERNAL_PROJECT_CMAKE_ARGS}
+             -DCMAKE_C_FLAGS=-w
+             -DCMAKE_CXX_FLAGS=-w
+             -DBENCHMARK_ENABLE_TESTING=OFF
+             -DBENCHMARK_ENABLE_GTEST_TESTS=OFF
+             -DBENCHMARK_ENABLE_WERROR=OFF
+             -DBENCHMARK_ENABLE_INSTALL=ON
+             -DBENCHMARK_ENABLE_LIBPFM=OFF
+  BUILD_COMMAND
+    "${CMAKE_COMMAND}" --build "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build"
+    --config $<CONFIG> --parallel
+  INSTALL_COMMAND
+    "${CMAKE_COMMAND}" --install
+    "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build" --config $<CONFIG>
+    --prefix "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/install"
+    ${PPC_EXTERNAL_PROJECT_LOG_ARGS})
+
+function(ppc_link_benchmark target_name)
+  target_include_directories(
+    ${target_name} PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/benchmark/include)
+
+  add_dependencies(${target_name} ppc_benchmark)
+  target_link_directories(${target_name} PUBLIC
+                          "${CMAKE_BINARY_DIR}/ppc_benchmark/install/lib")
+  target_link_libraries(${target_name} PUBLIC benchmark Threads::Threads)
+endfunction()

cmake/benchmark.cmake

@@ -1,11 +1,13 @@
 #pragma once
 
+#include <benchmark/benchmark.h>
 #include <gtest/gtest.h>
 #include <omp.h>
 #include <tbb/tick_count.h>
 
 #include <chrono>
 #include <cstddef>
+#include <cstdlib>
 #include <functional>
 #include <sstream>
 #include <stdexcept>
@@ -21,8 +23,64 @@
 
 namespace ppc::util {
 
-double GetTimeMPI();
-int GetMPIRank();
+namespace detail {
+
+inline bool ContainsFilterToken(std::string_view value, const char *filter_env) {
+  if (filter_env == nullptr || std::string_view(filter_env).empty()) {
+    return true;
+  }
+  return value.find(filter_env) != std::string_view::npos;
+}
+
+inline bool ShouldRunBenchmark(std::string_view test_name) {
+  return ContainsFilterToken(test_name, std::getenv("PPC_PERF_IMPL_FILTER")) &&
+         ContainsFilterToken(test_name, std::getenv("PPC_PERF_CATEGORY_FILTER"));
+}
+
+inline void CheckPerfMode(ppc::performance::PerfResults::TypeOfRunning mode) {
+  if (mode == ppc::performance::PerfResults::TypeOfRunning::kPipeline ||
+      mode == ppc::performance::PerfResults::TypeOfRunning::kTaskRun) {
+    return;
+  }
+  std::stringstream err_msg;
+  err_msg << '\n' << "The type of performance check for the task was not selected.\n";
+  throw std::runtime_error(err_msg.str().c_str());
+}
+
+template <typename InType, typename OutType>
+void RunTaskPipeline(const ppc::task::TaskPtr<InType, OutType> &task) {
+  task->Validation();
+  task->PreProcessing();
+  task->Run();
+  task->PostProcessing();
+}
+
+template <typename InType, typename OutType>
+void RunTaskForBenchmark(const ppc::task::TaskPtr<InType, OutType> &task,
+                         ppc::performance::PerfResults::TypeOfRunning mode, benchmark::State &state) {
+  task->GetStateOfTesting() = ppc::task::StateOfTesting::kPerf;
+  if (mode == ppc::performance::PerfResults::TypeOfRunning::kPipeline) {
+    SynchronizeMpiRanks();
+    state.ResumeTiming();
+    RunTaskPipeline(task);
+    state.PauseTiming();
+    return;
+  }
+
+  task->Validation();
+  task->PreProcessing();
+  SynchronizeMpiRanks();
+  state.ResumeTiming();
+  task->Run();
+  state.PauseTiming();
+  task->PostProcessing();
+}
+
+inline std::string MakeBenchmarkName(const std::string &test_name, ppc::performance::PerfResults::TypeOfRunning mode) {
+  return test_name + "/" + ppc::performance::GetStringParamName(mode);
+}
+
+}  // namespace detail
 
 template <typename InType, typename OutType>
 using PerfTestParam = std::tuple<std::function<ppc::task::TaskPtr<InType, OutType>(InType)>, std::string,
@@ -80,31 +138,35 @@ class BaseRunPerfTests : public ::testing::TestWithParam<PerfTestParam<InType, O
       // A single perf test body may execute several implementations; do not abort the enabled ones.
       return;
     }
+    if (!detail::ShouldRunBenchmark(test_name)) {
+      return;
+    }
+    detail::CheckPerfMode(mode);
 
     const auto test_env_scope = ppc::util::test::MakePerTestEnvForCurrentGTest(test_name);
 
-    task_ = task_getter(GetTestInputData());
-    ppc::performance::Perf perf(task_);
-    ppc::performance::PerfAttr perf_attr;
+    const auto input_data = GetTestInputData();
+    task_ = task_getter(input_data);
     SynchronizeMpiRanks();
-    SetPerfAttributes(perf_attr);
-
-    if (mode == ppc::performance::PerfResults::TypeOfRunning::kPipeline) {
-      perf.PipelineRun(perf_attr);
-    } else if (mode == ppc::performance::PerfResults::TypeOfRunning::kTaskRun) {
-      perf.TaskRun(perf_attr);
-    } else {
-      std::stringstream err_msg;
-      err_msg << '\n' << "The type of performance check for the task was not selected.\n";
-      throw std::runtime_error(err_msg.str().c_str());
-    }
-
-    if (GetMPIRank() == 0) {
-      perf.PrintPerfStatistic(test_name);
-    }
+    detail::RunTaskPipeline(task_);
 
     OutType output_data = task_->GetOutput();
     ASSERT_TRUE(CheckTestOutputData(output_data));
+
+    const auto benchmark_name = detail::MakeBenchmarkName(test_name, mode);
+    benchmark::RegisterBenchmark(benchmark_name,
+                                 [task_getter, input_data, mode](benchmark::State &state) {
+      for (auto _ : state) {
+        state.PauseTiming();
+        auto task = task_getter(input_data);
+        detail::RunTaskForBenchmark(task, mode, state);
+        benchmark::DoNotOptimize(task->GetOutput());
+        state.ResumeTiming();
+      }
+    })
+        ->UseRealTime()
+        ->Unit(benchmark::kMillisecond)
+        ->MinTime(0.01);
   }
 
  private:

modules/util/include/perf_test_util.hpp

@@ -75,6 +75,8 @@ int GetNumThreads();
 int GetNumProc();
 double GetTaskMaxTime();
 double GetPerfMaxTime();
+double GetTimeMPI();
+int GetMPIRank();
 void SynchronizeMpiRanks();
 
 template <typename T>

modules/util/include/util.hpp

@@ -1,6 +1,6 @@
 #include <mpi.h>
 
-#include "util/include/perf_test_util.hpp"
+#include "util/include/util.hpp"
 
 double ppc::util::GetTimeMPI() {
   return MPI_Wtime();