Use Google Benchmark for performance tables

Author: allnes

.editorconfig

@@ -17,6 +17,9 @@ indent_size = 2
 [*.py]
 indent_size = 4
 
+[*.{bat,cmd}]
+end_of_line = crlf
+
 [Makefile]
 indent_style = tab
 

.github/workflows/perf.yml

@@ -30,7 +30,7 @@ jobs:
           tar -xzvf ubuntu-gcc-install-ubuntu-24.04.tar.gz -C install
       - name: Run perf tests
         run: |
-          bash -e scripts/generate_perf_results.sh
+          scripts/run_tests.py --running-type=performance
         env:
           PPC_NUM_PROC: 2
           PPC_NUM_THREADS: 2
@@ -68,7 +68,7 @@ jobs:
           tar -xzvf macos-clang-install.tar.gz -C install
       - name: Run perf tests
         run: |
-          bash -e scripts/generate_perf_results.sh
+          scripts/run_tests.py --running-type=performance
         env:
           PPC_NUM_PROC: 1
           PPC_NUM_THREADS: 2

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "3rdparty/libenvpp"]
 	path = 3rdparty/libenvpp
 	url = https://github.com/ph3at/libenvpp
+[submodule "3rdparty/benchmark"]
+	path = 3rdparty/benchmark
+	url = https://github.com/google/benchmark

3rdparty/benchmark

@@ -44,6 +44,7 @@ endforeach()
 
 message( STATUS "PPC step: Setup external projects" )
 include(cmake/gtest.cmake)
+include(cmake/benchmark.cmake)
 
 ############################## Modules ##############################
 

CMakeLists.txt

@@ -0,0 +1,51 @@
+include_guard()
+
+include(ExternalProject)
+
+ExternalProject_Add(
+  ppc_benchmark
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/benchmark"
+  PREFIX "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark"
+  BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build"
+  INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/install"
+  EXCLUDE_FROM_ALL TRUE
+  CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+             -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+             -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
+             -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
+             -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
+             -DCMAKE_CXX_STANDARD_REQUIRED=${CMAKE_CXX_STANDARD_REQUIRED}
+             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+             ${PPC_EXTERNAL_PROJECT_CMAKE_ARGS}
+             -DCMAKE_C_FLAGS=-w
+             -DCMAKE_CXX_FLAGS=-w
+             -DBENCHMARK_ENABLE_TESTING=OFF
+             -DBENCHMARK_ENABLE_GTEST_TESTS=OFF
+             -DBENCHMARK_ENABLE_WERROR=OFF
+             -DBENCHMARK_ENABLE_INSTALL=ON
+             -DBENCHMARK_ENABLE_LIBPFM=OFF
+  BUILD_COMMAND
+    "${CMAKE_COMMAND}" --build "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build"
+    --config $<CONFIG> --parallel
+  INSTALL_COMMAND
+    "${CMAKE_COMMAND}" --install
+    "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/build" --config $<CONFIG>
+    --prefix "${CMAKE_CURRENT_BINARY_DIR}/ppc_benchmark/install"
+    ${PPC_EXTERNAL_PROJECT_LOG_ARGS})
+
+function(ppc_include_benchmark target_name)
+  target_include_directories(
+    ${target_name} PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty/benchmark/include)
+  target_compile_definitions(${target_name} PUBLIC BENCHMARK_STATIC_DEFINE)
+endfunction()
+
+function(ppc_link_benchmark target_name)
+  ppc_include_benchmark(${target_name})
+  add_dependencies(${target_name} ppc_benchmark)
+  target_link_directories(${target_name} PUBLIC
+                          "${CMAKE_BINARY_DIR}/ppc_benchmark/install/lib")
+  target_link_libraries(${target_name} PUBLIC benchmark Threads::Threads)
+  if(WIN32)
+    target_link_libraries(${target_name} PUBLIC shlwapi)
+  endif()
+endfunction()

cmake/benchmark.cmake

@@ -26,6 +26,7 @@ set_target_properties(${exec_func_lib} PROPERTIES LINKER_LANGUAGE CXX)
 target_include_directories(
   ${exec_func_lib} PUBLIC ${CMAKE_SOURCE_DIR}/3rdparty
                           ${CMAKE_SOURCE_DIR}/modules ${CMAKE_SOURCE_DIR}/tasks)
+ppc_include_benchmark(${exec_func_lib})
 
 foreach(
   link

modules/CMakeLists.txt

@@ -5,7 +5,6 @@
 #include <iomanip>
 #include <iostream>
 #include <memory>
-#include <sstream>
 #include <stdexcept>
 #include <string>
 
@@ -30,6 +29,7 @@ struct PerfAttr {
 struct PerfResults {
   /// @brief Measured execution time in seconds.
   double time_sec = 0.0;
+  bool is_measured = false;
   enum class TypeOfRunning : uint8_t {
     kPipeline,
     kTaskRun,
@@ -39,18 +39,26 @@ struct PerfResults {
   constexpr static double kMaxTime = 10.0;
 };
 
+inline std::string GetStringParamName(PerfResults::TypeOfRunning type_of_running) {
+  if (type_of_running == PerfResults::TypeOfRunning::kTaskRun) {
+    return "task_run";
+  }
+  if (type_of_running == PerfResults::TypeOfRunning::kPipeline) {
+    return "pipeline";
+  }
+  return "none";
+}
+
 template <typename InType, typename OutType>
 class Perf {
  public:
   // Init performance analysis with an initialized task and initialized data
   explicit Perf(const ppc::task::TaskPtr<InType, OutType> &task_ptr) : task_(task_ptr) {
     task_ptr->GetStateOfTesting() = ppc::task::StateOfTesting::kPerf;
   }
-  // Check performance of full task's pipeline:  PreProcessing() ->
-  // Validation() -> Run() -> PostProcessing()
+  // Check performance of the full task pipeline. Kept for source compatibility with older student tests.
   void PipelineRun(const PerfAttr &perf_attr) {
     perf_results_.type_of_running = PerfResults::TypeOfRunning::kPipeline;
-
     CommonRun(perf_attr, [&] {
       task_->Validation();
       task_->PreProcessing();
@@ -61,7 +69,6 @@ class Perf {
   // Check performance of task's Run() function
   void TaskRun(const PerfAttr &perf_attr) {
     perf_results_.type_of_running = PerfResults::TypeOfRunning::kTaskRun;
-
     task_->Validation();
     task_->PreProcessing();
     CommonRun(perf_attr, [&] { task_->Run(); }, perf_results_);
@@ -74,31 +81,21 @@ class Perf {
   }
   // Print results for automation checkers
   void PrintPerfStatistic(const std::string &test_id) const {
-    std::string type_test_name;
-    if (perf_results_.type_of_running == PerfResults::TypeOfRunning::kTaskRun) {
-      type_test_name = "task_run";
-    } else if (perf_results_.type_of_running == PerfResults::TypeOfRunning::kPipeline) {
-      type_test_name = "pipeline";
-    } else {
-      std::stringstream err_msg;
-      err_msg << '\n' << "The type of performance check for the task was not selected.\n";
-      throw std::runtime_error(err_msg.str().c_str());
+    if (!perf_results_.is_measured) {
+      throw std::runtime_error("Performance measurement was not run.");
+    }
+    if (perf_results_.type_of_running == PerfResults::TypeOfRunning::kNone) {
+      throw std::runtime_error("The type of performance check for the task was not selected.");
     }
 
     auto time_secs = perf_results_.time_sec;
     const auto max_time = ppc::util::GetPerfMaxTime();
-    std::stringstream perf_res_str;
+    const auto type_test_name = GetStringParamName(perf_results_.type_of_running);
     if (time_secs < max_time) {
-      perf_res_str << std::fixed << std::setprecision(10) << time_secs;
-      std::cout << test_id << ":" << type_test_name << ":" << perf_res_str.str() << '\n';
+      std::cout << test_id << ':' << type_test_name << ':' << std::fixed << std::setprecision(10) << time_secs << '\n';
     } else {
-      std::stringstream err_msg;
-      err_msg << '\n' << "Task execute time need to be: ";
-      err_msg << "time < " << max_time << " secs." << '\n';
-      err_msg << "Original time in secs: " << time_secs << '\n';
-      perf_res_str << std::fixed << std::setprecision(10) << -1.0;
-      std::cout << test_id << ":" << type_test_name << ":" << perf_res_str.str() << '\n';
-      throw std::runtime_error(err_msg.str().c_str());
+      std::cout << test_id << ':' << type_test_name << ':' << std::fixed << std::setprecision(10) << -1.0 << '\n';
+      throw std::runtime_error("Task execution time exceeded the performance limit.");
     }
   }
   /// @brief Retrieves the performance test results.
@@ -110,24 +107,15 @@ class Perf {
  private:
   PerfResults perf_results_;
   std::shared_ptr<ppc::task::Task<InType, OutType>> task_;
-  static void CommonRun(const PerfAttr &perf_attr, const std::function<void()> &pipeline, PerfResults &perf_results) {
+  static void CommonRun(const PerfAttr &perf_attr, const std::function<void()> &operation, PerfResults &perf_results) {
     auto begin = perf_attr.current_timer();
     for (uint64_t i = 0; i < perf_attr.num_running; i++) {
-      pipeline();
+      operation();
     }
     auto end = perf_attr.current_timer();
     perf_results.time_sec = (end - begin) / static_cast<double>(perf_attr.num_running);
+    perf_results.is_measured = true;
   }
 };
 
-inline std::string GetStringParamName(PerfResults::TypeOfRunning type_of_running) {
-  if (type_of_running == PerfResults::TypeOfRunning::kTaskRun) {
-    return "task_run";
-  }
-  if (type_of_running == PerfResults::TypeOfRunning::kPipeline) {
-    return "pipeline";
-  }
-  return "none";
-}
-
 }  // namespace ppc::performance

modules/performance/include/performance.hpp

@@ -68,37 +68,37 @@ class FakePerfTask : public TestPerfTask<InType, OutType> {
 
 namespace ppc::performance {
 
-TEST(PerfTests, CheckPerfPipeline) {
+TEST(PerfTests, CheckPerfTaskRun) {
   std::vector<uint32_t> in(2000, 1);
 
   auto test_task = std::make_shared<ppc::test::TestPerfTask<std::vector<uint32_t>, uint32_t>>(in);
 
   Perf<std::vector<uint32_t>, uint32_t> perf_analyzer(test_task);
 
   PerfAttr perf_attr;
-  perf_analyzer.PipelineRun(perf_attr);
+  perf_analyzer.TaskRun(perf_attr);
 
-  perf_analyzer.PrintPerfStatistic("check_perf_pipeline");
+  perf_analyzer.PrintPerfStatistic("check_perf");
   ASSERT_LE(perf_analyzer.GetPerfResults().time_sec, PerfResults::kMaxTime);
   EXPECT_EQ(test_task->GetOutput(), in.size());
 }
 
-TEST(PerfTests, CheckPerfPipelineFloat) {
+TEST(PerfTests, CheckPerfTaskRunFloat) {
   std::vector<float> in(2000, 1);
 
   auto test_task = std::make_shared<ppc::test::TestPerfTask<std::vector<float>, float>>(in);
 
   Perf<std::vector<float>, float> perf_analyzer(test_task);
 
   PerfAttr perf_attr;
-  perf_analyzer.PipelineRun(perf_attr);
+  perf_analyzer.TaskRun(perf_attr);
 
-  perf_analyzer.PrintPerfStatistic("check_perf_pipeline_float");
+  perf_analyzer.PrintPerfStatistic("check_perf_float");
   ASSERT_LE(perf_analyzer.GetPerfResults().time_sec, PerfResults::kMaxTime);
   EXPECT_EQ(test_task->GetOutput(), in.size());
 }
 
-TEST(PerfTests, CheckPerfPipelineUint8tSlowTest) {
+TEST(PerfTests, CheckPerfUint8tSlowTest) {
   std::vector<uint8_t> in(128, 1);
 
   auto test_task = std::make_shared<ppc::test::FakePerfTask<std::vector<uint8_t>, uint8_t>>(in);
@@ -114,9 +114,9 @@ TEST(PerfTests, CheckPerfPipelineUint8tSlowTest) {
     auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(current_time_point - t0).count();
     return static_cast<double>(duration) * 1e-9;
   };
-  perf_analyzer.PipelineRun(perf_attr);
+  perf_analyzer.TaskRun(perf_attr);
 
-  ASSERT_ANY_THROW(perf_analyzer.PrintPerfStatistic("check_perf_pipeline_uint8_t_slow_test"));
+  ASSERT_ANY_THROW(perf_analyzer.PrintPerfStatistic("check_perf_uint8_t_slow_test"));
 }
 
 TEST(PerfTests, SlowPerfRespectsEnvOverride) {
@@ -132,7 +132,7 @@ TEST(PerfTests, SlowPerfRespectsEnvOverride) {
     auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(current_time_point - t0).count();
     return static_cast<double>(duration) * 1e-9;
   };
-  perf_analyzer.PipelineRun(perf_attr);
+  perf_analyzer.TaskRun(perf_attr);
   EXPECT_NO_THROW(perf_analyzer.PrintPerfStatistic("slow_perf_respects_env_override"));
 }
 
@@ -164,26 +164,6 @@ TEST(PerfTests, CheckPerfTaskFloat) {
   EXPECT_EQ(test_task->GetOutput(), in.size());
 }
 
-struct ParamTestCase {
-  PerfResults::TypeOfRunning input;
-  std::string_view expected_output;
-};
-
-namespace {
-
-constexpr std::array<ParamTestCase, 3> kParamTestCases = {
-    {{.input = PerfResults::TypeOfRunning::kTaskRun, .expected_output = "task_run"},
-     {.input = PerfResults::TypeOfRunning::kPipeline, .expected_output = "pipeline"},
-     {.input = PerfResults::TypeOfRunning::kNone, .expected_output = "none"}}};
-
-}  // namespace
-
-TEST(GetStringParamNameParamTest, ReturnsExpectedString) {
-  for (const auto &param : kParamTestCases) {
-    EXPECT_EQ(GetStringParamName(param.input), std::string(param.expected_output));
-  }
-}
-
 struct TaskTypeTestCase {
   TypeOfTask type;
   std::string_view expected;
@@ -336,7 +316,7 @@ TEST(GetNamespaceTest, ReturnsEmptyStringForGlobalNamespaceType) {
   EXPECT_EQ(ppc::util::GetNamespace<int>(), "");
 }
 
-TEST(PerfTest, PipelineRunAndTaskRun) {
+TEST(PerfTest, TaskRunMeasuresExecutionTime) {
   auto task_ptr = std::make_shared<DummyTask>();
   Perf<int, int> perf(task_ptr);
 
@@ -349,15 +329,31 @@ TEST(PerfTest, PipelineRunAndTaskRun) {
     return t;
   };
 
-  EXPECT_NO_THROW(perf.PipelineRun(attr));
-  auto res_pipeline = perf.GetPerfResults();
-  EXPECT_EQ(res_pipeline.type_of_running, PerfResults::TypeOfRunning::kPipeline);
-  EXPECT_GT(res_pipeline.time_sec, 0.0);
-
   EXPECT_NO_THROW(perf.TaskRun(attr));
-  auto res_taskrun = perf.GetPerfResults();
-  EXPECT_EQ(res_taskrun.type_of_running, PerfResults::TypeOfRunning::kTaskRun);
-  EXPECT_GT(res_taskrun.time_sec, 0.0);
+  auto result = perf.GetPerfResults();
+  EXPECT_TRUE(result.is_measured);
+  EXPECT_GT(result.time_sec, 0.0);
+}
+
+TEST(PerfTest, PipelineRunCompatibilityApiMeasuresExecutionTime) {
+  auto task_ptr = std::make_shared<DummyTask>();
+  Perf<int, int> perf(task_ptr);
+
+  PerfAttr attr;
+  double time = 0.0;
+  attr.num_running = 2;
+  attr.current_timer = [&time]() {
+    double t = time;
+    time += 1.0;
+    return t;
+  };
+
+  perf.PipelineRun(attr);
+  auto result = perf.GetPerfResults();
+  EXPECT_TRUE(result.is_measured);
+  EXPECT_EQ(result.type_of_running, PerfResults::TypeOfRunning::kPipeline);
+  EXPECT_GT(result.time_sec, 0.0);
+  EXPECT_EQ(GetStringParamName(result.type_of_running), "pipeline");
 }
 
 TEST(PerfTest, PrintPerfStatisticThrowsOnNone) {
@@ -370,12 +366,6 @@ TEST(PerfTest, PrintPerfStatisticThrowsOnNone) {
   ppc::util::DestructorFailureFlag::Unset();
 }
 
-TEST(PerfTest, GetStringParamNameTest) {
-  EXPECT_EQ(GetStringParamName(PerfResults::TypeOfRunning::kTaskRun), "task_run");
-  EXPECT_EQ(GetStringParamName(PerfResults::TypeOfRunning::kPipeline), "pipeline");
-  EXPECT_EQ(GetStringParamName(PerfResults::TypeOfRunning::kNone), "none");
-}
-
 TEST(TaskTest, DestructorInvalidPipelineOrderTerminatesPartialPipeline) {
   {
     struct BadTask : Task<int, int> {