Using RuntimeOptions to set up the graph (#242)

close #226

---------

Co-authored-by: Arseniy Obolenskiy <gooddoog@student.su>

Author: Semyon1104

app/Accuracy/accuracy_check.cpp

@@ -10,6 +10,10 @@
 using namespace it_lab_ai;
 
 int main() {
+  RuntimeOptions options;
+  options.backend = Backend::kNaive;
+  options.threads = 4;
+  options.parallel = true;
   std::string image_path = IMAGE1_PATH;
   cv::Mat image = cv::imread(image_path);
   if (image.empty()) {
@@ -66,7 +70,7 @@ int main() {
   graph.makeConnection(a4_ptr, a5_ptr);
   graph.makeConnection(a5_ptr, a6_ptr);
   graph.setOutput(a5_ptr, output);
-  graph.inference();
+  graph.inference(options);
   std::vector<float> tmp = *output.as<float>();
   std::vector<float> tmp_output = softmax<float>(*output.as<float>());
   for (float i : tmp) {

app/Graph/acc_check.cpp

@@ -12,15 +12,45 @@ using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   std::string model_name = "alexnet_mnist";
-  bool onednn = false;
+  RuntimeOptions options;
+
   for (int i = 1; i < argc; ++i) {
     if (std::string(argv[i]) == "--model" && i + 1 < argc) {
       model_name = argv[++i];
     } else if (std::string(argv[i]) == "--onednn") {
-      onednn = true;
+      options.backend = Backend::kOneDnn;
+      if (options.isParallel()) {
+        std::cout << "Warning: oneDNN backend is not compatible with parallel "
+                     "execution. Disabling parallelism."
+                  << '\n';
+        options.setParallelBackend(ParBackend::kSeq);
+      }
+    } else if (std::string(argv[i]) == "--parallel" && i + 1 < argc) {
+      if (options.backend == Backend::kOneDnn) {
+        std::cout << "Warning: Parallel execution is not compatible with "
+                     "oneDNN backend. Ignoring --parallel option."
+                  << '\n';
+        i++;
+        continue;
+      }
+
+      std::string backend_str = argv[++i];
+      if (backend_str == "tbb") {
+        options.setParallelBackend(ParBackend::kTbb);
+      } else if (backend_str == "threads" || backend_str == "stl") {
+        options.setParallelBackend(ParBackend::kThreads);
+      } else if (backend_str == "omp") {
+        options.setParallelBackend(ParBackend::kOmp);
+      } else {
+        std::cerr << "Unknown parallel backend: " << backend_str
+                  << ". Using default (Threads)." << '\n';
+        options.setParallelBackend(ParBackend::kThreads);
+      }
+    } else if (std::string(argv[i]) == "--threads" && i + 1 < argc) {
+      options.threads = std::stoi(argv[++i]);
     }
   }
-  it_lab_ai::LayerFactory::configure(onednn);
+
   std::string dataset_path;
   if (model_name == "alexnet_mnist") {
     dataset_path = MNIST_PATH;
@@ -77,8 +107,8 @@ int main(int argc, char* argv[]) {
     Tensor t = make_tensor<float>(res, sh);
     input = t;
     Graph graph;
-    build_graph_linear(graph, input, output, false);
-    graph.inference();
+    build_graph_linear(graph, input, output, options, false);
+    graph.inference(options);
     print_time_stats(graph);
     std::vector<std::vector<float>> tmp_output =
         softmax<float>(*output.as<float>(), 10);
@@ -186,8 +216,8 @@ int main(int argc, char* argv[]) {
       it_lab_ai::Tensor(output_shape, it_lab_ai::Type::kFloat);
 
   Graph graph;
-  build_graph(graph, input, output, json_path, false);
-  graph.inference();
+  build_graph(graph, input, output, json_path, options, false);
+  graph.inference(options);
   print_time_stats(graph);
   std::vector<std::vector<float>> processed_outputs;
   const std::vector<float>& raw_output = *output.as<float>();

app/Graph/build.cpp

@@ -4,8 +4,6 @@
 
 using namespace it_lab_ai;
 
-bool LayerFactory::onednn_ = false;
-
 std::unordered_map<std::string, std::string> model_paths = {
     {"alexnet_mnist", MODEL_PATH_H5},
     {"googlenet", MODEL_PATH_GOOGLENET_ONNX},
@@ -14,7 +12,8 @@ std::unordered_map<std::string, std::string> model_paths = {
     {"yolo", MODEL_PATH_YOLO11NET_ONNX}};
 
 void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
-                        it_lab_ai::Tensor& output, bool comments) {
+                        it_lab_ai::Tensor& output, RuntimeOptions options,
+                        bool comments) {
   if (comments) {
     for (size_t i = 0; i < input.get_shape().dims(); i++) {
       std::cout << input.get_shape()[i] << ' ';
@@ -83,14 +82,14 @@ void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
       it_lab_ai::Tensor tmp_values = tensor;
       it_lab_ai::Tensor tmp_bias = it_lab_ai::make_tensor(tensor.get_bias());
       auto conv_layer = std::make_unique<it_lab_ai::ConvolutionalLayer>(
-          1, pads, 1, tmp_values, tmp_bias, kDefault, 1, true);
+          1, pads, 1, tmp_values, tmp_bias, 1, true);
       layer_ptrs.push_back(conv_layer.get());
       layers.push_back(std::move(conv_layer));
       layerpostop.push_back(false);
       if (comments) std::cout << "ConvLayer added to layers." << '\n';
     }
     if (layer_type.find("relu") != std::string::npos) {
-      auto ew_layer = LayerFactory::createEwLayer("relu");
+      auto ew_layer = LayerFactory::createEwLayer("relu", options);
       layer_ptrs.push_back(ew_layer.get());
       layers.push_back(std::move(ew_layer));
       layerpostop.push_back(true);
@@ -120,7 +119,7 @@ void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                   << '\n';
       }
       auto pool_layer =
-          std::make_unique<it_lab_ai::PoolingLayer>(shape, pooltype, kDefault);
+          std::make_unique<it_lab_ai::PoolingLayer>(shape, pooltype);
       layer_ptrs.push_back(pool_layer.get());
       layers.push_back(std::move(pool_layer));
       layerpostop.push_back(false);
@@ -195,8 +194,8 @@ std::string get_base_layer_name(const std::string& tensor_name) {
 
 void build_graph(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                  it_lab_ai::Tensor& output, const std::string& json_path,
-                 bool comments) {
-  auto parse_result = parse_json_model(json_path, comments);
+                 RuntimeOptions options, bool comments) {
+  auto parse_result = parse_json_model(options, json_path, comments);
 
   auto& layers = parse_result.layers;
   auto& name_to_layer_ptr = parse_result.name_to_layer_ptr;
@@ -300,7 +299,8 @@ void build_graph(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
   }
 }
 
-ParseResult parse_json_model(const std::string& json_path, bool comments) {
+ParseResult parse_json_model(RuntimeOptions options,
+                             const std::string& json_path, bool comments) {
   ParseResult result;
 
   auto& layers = result.layers;
@@ -411,13 +411,13 @@ ParseResult parse_json_model(const std::string& json_path, bool comments) {
         it_lab_ai::Tensor tmp_bias = it_lab_ai::make_tensor(tensor.get_bias());
 
         auto conv_layer = std::make_unique<it_lab_ai::ConvolutionalLayer>(
-            stride, pads, dilations, tmp_tensor, tmp_bias, kDefault, group);
+            stride, pads, dilations, tmp_tensor, tmp_bias, group);
         layer = std::move(conv_layer);
       } else if (layer_type.find("Relu") != std::string::npos ||
                  layer_type.find("relu") != std::string::npos) {
-        layer = LayerFactory::createEwLayer("relu");
+        layer = LayerFactory::createEwLayer("relu", options);
       } else if (layer_type.find("Sigmoid") != std::string::npos) {
-        layer = LayerFactory::createEwLayer("sigmoid");
+        layer = LayerFactory::createEwLayer("sigmoid", options);
       } else if (layer_type.find("Dense") != std::string::npos ||
                  layer_type.find("FullyConnected") != std::string::npos) {
         it_lab_ai::Tensor tensor = it_lab_ai::create_tensor_from_json(
@@ -448,7 +448,7 @@ ParseResult parse_json_model(const std::string& json_path, bool comments) {
         }
       } else if (layer_type == "GlobalAveragePool") {
         auto pool_layer = std::make_unique<it_lab_ai::PoolingLayer>(
-            it_lab_ai::Shape({0, 0}), "average", kDefault);
+            it_lab_ai::Shape({0, 0}), "average");
         layer = std::move(pool_layer);
         if (comments) {
           std::cout << "GlobalAveragePool layer added (will use input spatial "
@@ -509,8 +509,8 @@ ParseResult parse_json_model(const std::string& json_path, bool comments) {
           }
         }
 
-        auto pool_layer = std::make_unique<it_lab_ai::PoolingLayer>(
-            shape, pooltype, kDefault);
+        auto pool_layer =
+            std::make_unique<it_lab_ai::PoolingLayer>(shape, pooltype);
 
         try {
           if (strides[0] != 2 || strides[1] != 2) {
@@ -637,13 +637,16 @@ ParseResult parse_json_model(const std::string& json_path, bool comments) {
 
           if (layer_type == "Mul") {
             ew_operation = "linear";
-            layer = LayerFactory::createEwLayer(ew_operation, value, 0.0F);
+            layer =
+                LayerFactory::createEwLayer(ew_operation, options, value, 0.0F);
           } else if (layer_type == "Add") {
             ew_operation = "linear";
-            layer = LayerFactory::createEwLayer(ew_operation, 1.0F, value);
+            layer =
+                LayerFactory::createEwLayer(ew_operation, options, 1.0F, value);
           } else if (layer_type == "Sub") {
             ew_operation = "linear";
-            layer = LayerFactory::createEwLayer(ew_operation, 1.0F, -value);
+            layer = LayerFactory::createEwLayer(ew_operation, options, 1.0F,
+                                                -value);
           } else {
             continue;
           }

app/Graph/build.hpp

@@ -13,6 +13,7 @@
 
 #include "Weights_Reader/reader_weights.hpp"
 #include "graph/graph.hpp"
+#include "graph/runtime_options.hpp"
 #include "layers/BatchNormalizationLayer.hpp"
 #include "layers/BinaryOpLayer.hpp"
 #include "layers/ConcatLayer.hpp"
@@ -52,13 +53,15 @@ struct ParseResult {
 
 void build_graph(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                  it_lab_ai::Tensor& output, const std::string& json_path,
-                 bool comments);
+                 it_lab_ai::RuntimeOptions options, bool comments);
 void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
-                        it_lab_ai::Tensor& output, bool comments);
+                        it_lab_ai::Tensor& output,
+                        it_lab_ai::RuntimeOptions options, bool comments);
 std::unordered_map<int, std::string> load_class_names(
     const std::string& filename);
 
-ParseResult parse_json_model(const std::string& json_path, bool comments);
+ParseResult parse_json_model(it_lab_ai::RuntimeOptions options,
+                             const std::string& json_path, bool comments);
 
 std::vector<int> get_input_shape_from_json(const std::string& json_path);
 std::vector<float> process_model_output(const std::vector<float>& output,
@@ -69,19 +72,15 @@ it_lab_ai::Tensor prepare_image(const cv::Mat& image,
 it_lab_ai::Tensor prepare_mnist_image(const cv::Mat& image);
 
 void print_time_stats(it_lab_ai::Graph& graph);
-
 namespace it_lab_ai {
 class LayerFactory {
- private:
-  static bool onednn_;
-
  public:
-  static void configure(bool onednn) { onednn_ = onednn; }
-
   static std::unique_ptr<Layer> createEwLayer(const std::string& function,
+                                              const RuntimeOptions& options,
                                               float alpha = 1.0F,
                                               float beta = 0.0F) {
-    if (onednn_ && EwLayerOneDnn::is_function_supported(function)) {
+    if (options.backend == Backend::kOneDnn &&
+        EwLayerOneDnn::is_function_supported(function)) {
       return std::make_unique<EwLayerOneDnn>(function, alpha, beta);
     }
     return std::make_unique<EWLayer>(function, alpha, beta);

app/Graph/graph_build.cpp

@@ -9,17 +9,45 @@ using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   std::string model_name = "alexnet_mnist";
-  bool onednn = false;
+  RuntimeOptions options;
+
   for (int i = 1; i < argc; ++i) {
     if (std::string(argv[i]) == "--model" && i + 1 < argc) {
       model_name = argv[++i];
     } else if (std::string(argv[i]) == "--onednn") {
-      onednn = true;
+      options.backend = Backend::kOneDnn;
+      if (options.isParallel()) {
+        std::cout << "Warning: oneDNN backend is not compatible with parallel "
+                     "execution. Disabling parallelism."
+                  << '\n';
+        options.setParallelBackend(ParBackend::kSeq);
+      }
+    } else if (std::string(argv[i]) == "--parallel" && i + 1 < argc) {
+      if (options.backend == Backend::kOneDnn) {
+        std::cout << "Warning: Parallel execution is not compatible with "
+                     "oneDNN backend. Ignoring --parallel option."
+                  << '\n';
+        i++;
+        continue;
+      }
+
+      std::string backend_str = argv[++i];
+      if (backend_str == "tbb") {
+        options.setParallelBackend(ParBackend::kTbb);
+      } else if (backend_str == "threads" || backend_str == "stl") {
+        options.setParallelBackend(ParBackend::kThreads);
+      } else if (backend_str == "omp") {
+        options.setParallelBackend(ParBackend::kOmp);
+      } else {
+        std::cerr << "Unknown parallel backend: " << backend_str
+                  << ". Using default (Threads)." << '\n';
+        options.setParallelBackend(ParBackend::kThreads);
+      }
+    } else if (std::string(argv[i]) == "--threads" && i + 1 < argc) {
+      options.threads = std::stoi(argv[++i]);
     }
   }
 
-  it_lab_ai::LayerFactory::configure(onednn);
-
   std::string json_path = model_paths[model_name];
 
   std::vector<int> input_shape;
@@ -62,11 +90,11 @@ int main(int argc, char* argv[]) {
         std::vector<float> vec(75, 3);
         it_lab_ai::Tensor output = it_lab_ai::make_tensor(vec, sh1);
         Graph graph;
-        build_graph_linear(graph, input, output, true);
+        build_graph_linear(graph, input, output, options, true);
 
         std::cout << "Starting inference..." << '\n';
         try {
-          graph.inference();
+          graph.inference(options);
           std::cout << "Inference completed successfully." << '\n';
         } catch (const std::exception& e) {
           std::cerr << "ERROR during inference: " << e.what() << '\n';
@@ -102,11 +130,11 @@ int main(int argc, char* argv[]) {
         it_lab_ai::Tensor output({1, output_classes}, it_lab_ai::Type::kFloat);
 
         Graph graph;
-        build_graph(graph, input, output, json_path, false);
+        build_graph(graph, input, output, json_path, options, false);
 
         std::cout << "Starting inference..." << '\n';
         try {
-          graph.inference();
+          graph.inference(options);
           std::cout << "Inference completed successfully." << '\n';
         } catch (const std::exception& e) {
           std::cerr << "ERROR during inference: " << e.what() << '\n';

include/graph/graph.hpp

@@ -11,6 +11,7 @@
 #include <vector>
 
 #include "layers/Layer.hpp"
+#include "runtime_options.hpp"
 
 namespace it_lab_ai {
 
@@ -224,7 +225,7 @@ class Graph {
     return false;
   }
 
-  void inference() {
+  void inference(const RuntimeOptions& options) {
     std::vector<std::pair<int, int>> countinout = getInOutDegrees();
     std::vector<int> traversal = getTraversalOrder();
     count_used_split_distribution_ = 0;
@@ -262,7 +263,7 @@ class Graph {
           }
         }
       }
-      layers_[current_layer]->run(inten_, outten_);
+      layers_[current_layer]->run(inten_, outten_, options);
 
 #ifdef ENABLE_STATISTIC_TENSORS
       tensors_.push_back(inten_[0]);
@@ -277,7 +278,8 @@ class Graph {
       if (layers_[current_layer]->postops.count > 0) {
         for (unsigned int j = 0; j < layers_[current_layer]->postops.count;
              j++) {
-          layers_[current_layer]->postops.layers[j]->run(inten_, outten_);
+          layers_[current_layer]->postops.layers[j]->run(inten_, outten_,
+                                                         options);
         }
         inten_ = outten_;
       }