Breaking down perf_tests into time and memory tests (#114)

This revision includes:
* Splitting the perf_tests app into time_perf_tests and mem_perf_tests
* Introducing Google Benchmark library for time_perf_tests
* API for changing number of threads

Author: doyubkim

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "external/pybind11"]
 	path = external/pybind11
 	url = https://github.com/pybind/pybind11.git
+[submodule "external/googlebenchmark"]
+	path = external/googlebenchmark
+	url = https://github.com/google/benchmark.git

3RD_PARTY.md

@@ -483,3 +483,22 @@ hereby grant the following license: a non-exclusive, royalty-free perpetual
 license to install, use, modify, prepare derivative works, incorporate into
 other computer software, distribute, and sublicense such enhancements or
 derivative works thereof, in binary and source code form.
+
+---
+
+Jet uses Google Benchmark for perf testing.
+
+Copyright 2017, Google Inc.
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

CMakeLists.txt

@@ -62,13 +62,16 @@ add_subdirectory(external/googletest EXCLUDE_FROM_ALL)
 if ((CMAKE_VERSION VERSION_EQUAL 3.3) OR (CMAKE_VERSION VERSION_GREATER 3.3))
     cmake_policy(SET CMP0063 NEW)
 endif()
+set(BENCHMARK_ENABLE_TESTING 0)
+add_subdirectory(external/googlebenchmark)
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 set(CMAKE_VISIBILITY_INLINES_HIDDEN 1)
 add_subdirectory(external/cnpy)
 add_subdirectory(external/pystring)
 add_subdirectory(src/jet)
 add_subdirectory(src/tests/manual_tests)
-add_subdirectory(src/tests/perf_tests)
+add_subdirectory(src/tests/mem_perf_tests)
+add_subdirectory(src/tests/time_perf_tests)
 add_subdirectory(src/tests/unit_tests)
 if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
     add_subdirectory(src/winix)

INSTALL.md

@@ -97,7 +97,7 @@ bin\Release\unit_tests.exe
 
 ### Running Tests
 
-There are four different tests in the codebase including the unit test, manual test, performance test, and Python API test. For the detailed instruction on how to run those tests, please checkout the documentation page from [the project website](http://doyubkim.github.io/fluid-engine-dev/documentation/).
+There are four different tests in the codebase including the unit test, manual test, time/memory performance tests, and Python API test. For the detailed instruction on how to run those tests, please checkout the documentation page from [the project website](http://doyubkim.github.io/fluid-engine-dev/documentation/).
 
 ### Installing C++ SDK
 

cmake/CompileOptions.cmake

@@ -72,6 +72,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
         # /wd4592       # -> disable warning: 'identifier': symbol will be dynamically initialized (implementation limitation)
         # /wd4201     # -> disable warning: nonstandard extension used: nameless struct/union (caused by GLM)
         # /wd4127     # -> disable warning: conditional expression is constant (caused by Qt)
+        /wd4717 # -> disable warning:  recursive on all control paths, function will cause runtime stack overflow (wrong warning)
 
         #$<$<CONFIG:Debug>:
         #/RTCc         # -> value is assigned to a smaller data type and results in a data loss

external/googlebenchmark

@@ -116,8 +116,7 @@ void parallelFor(IndexType start, IndexType end, const Function& func,
     }
 
     // Estimate number of threads in the pool
-    static const unsigned int numThreadsHint =
-        std::thread::hardware_concurrency();
+    unsigned int numThreadsHint = maxNUmberOfThreads();
     const unsigned int numThreads =
         (policy == ExecutionPolicy::kParallel)
             ? (numThreadsHint == 0u ? 8u : numThreadsHint)
@@ -166,8 +165,7 @@ void parallelRangeFor(IndexType start, IndexType end, const Function& func,
     }
 
     // Estimate number of threads in the pool
-    static const unsigned int numThreadsHint =
-        std::thread::hardware_concurrency();
+    unsigned int numThreadsHint = maxNUmberOfThreads();
     const unsigned int numThreads =
         (policy == ExecutionPolicy::kParallel)
             ? (numThreadsHint == 0u ? 8u : numThreadsHint)
@@ -264,8 +262,7 @@ Value parallelReduce(IndexType start, IndexType end, const Value& identity,
     }
 
     // Estimate number of threads in the pool
-    static const unsigned int numThreadsHint =
-        std::thread::hardware_concurrency();
+    unsigned int numThreadsHint = maxNUmberOfThreads();
     const unsigned int numThreads =
         (policy == ExecutionPolicy::kParallel)
             ? (numThreadsHint == 0u ? 8u : numThreadsHint)
@@ -282,7 +279,6 @@ Value parallelReduce(IndexType start, IndexType end, const Value& identity,
 
     // [Helper] Inner loop
     auto launchRange = [&](IndexType k1, IndexType k2, unsigned int tid) {
-        // TODO: This is bad in terms of performance!
         results[tid] = func(k1, k2, identity);
     };
 
@@ -331,8 +327,7 @@ void parallelSort(RandomIterator begin, RandomIterator end,
     std::vector<value_type> temp(size);
 
     // Estimate number of threads in the pool
-    static const unsigned int numThreadsHint =
-        std::thread::hardware_concurrency();
+    unsigned int numThreadsHint = maxNUmberOfThreads();
     const unsigned int numThreads =
         (policy == ExecutionPolicy::kParallel)
             ? (numThreadsHint == 0u ? 8u : numThreadsHint)

include/jet/detail/parallel-inl.h

@@ -237,6 +237,12 @@ void parallelSort(RandomIterator begin, RandomIterator end,
                   CompareFunction compare,
                   ExecutionPolicy policy = ExecutionPolicy::kParallel);
 
+//! Sets maximum number of threads to use.
+void setMaxNumberOfThreads(unsigned int numThreads);
+
+//! Returns maximum number of threads to use.
+unsigned int maxNUmberOfThreads();
+
 }  // namespace jet
 
 #include "detail/parallel-inl.h"

include/jet/parallel.h

@@ -0,0 +1,23 @@
+// Copyright (c) 2017 Doyub Kim
+//
+// I am making my contributions/submissions to this project solely in my
+// personal capacity and am not conveying any rights to any intellectual
+// property of any third parties.
+
+#include <jet/parallel.h>
+
+#include <thread>
+
+static unsigned int sMaxNumberOfThreads = std::thread::hardware_concurrency();
+
+namespace jet {
+
+void setMaxNumberOfThreads(unsigned int numThreads) {
+    sMaxNumberOfThreads = std::max(numThreads, 1u);
+}
+
+unsigned int maxNUmberOfThreads() {
+    return sMaxNumberOfThreads;
+}
+
+}  // namespace jet

src/jet/parallel.cpp

@@ -7,7 +7,7 @@
 #
 
 # Target name
-set(target perf_tests)
+set(target mem_perf_tests)
 
 # Includes
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})