Fuse Convolution + Activation node as a part of graph transformations pipeline (#276)

Author: NeiroYT

app/Graph/CMakeLists.txt

@@ -11,6 +11,7 @@ target_link_libraries(BuildGraph PUBLIC ${OpenCV_LIBS})
 target_link_libraries(BuildGraph PUBLIC reader_lib)
 target_link_libraries(BuildGraph PUBLIC TBB_unified)
 target_link_libraries(BuildGraph PUBLIC layers_lib)
+target_link_libraries(BuildGraph PUBLIC layers_fused_lib)
 target_link_libraries(BuildGraph PUBLIC layers_oneDNN_lib)
 target_link_libraries(BuildGraph PUBLIC gtest_main)
 
@@ -25,6 +26,7 @@ target_link_libraries(ACC BuildGraph)
 add_executable(onnx_subgraphs onnx_subgraphs.cpp)
 target_link_libraries(onnx_subgraphs BuildGraph)
 target_link_libraries(onnx_subgraphs OpenMP::OpenMP_CXX)
+target_link_libraries(onnx_subgraphs ${OpenCV_LIBS})
 target_link_libraries(onnx_subgraphs graphT_lib)
 
 file(DOWNLOAD

app/Graph/build.cpp

@@ -13,7 +13,7 @@ std::unordered_map<std::string, std::string> model_paths = {
 
 void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                         it_lab_ai::Tensor& output, RuntimeOptions options,
-                        bool comments) {
+                        bool comments, bool enable_postops) {
   if (comments) {
     for (size_t i = 0; i < input.get_shape().dims(); i++) {
       std::cout << input.get_shape()[i] << ' ';
@@ -75,7 +75,7 @@ void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
     if (layer_type.find("relu") != std::string::npos) {
       auto ew_layer = LayerFactory::createEwLayer("relu", options);
       layers.push_back(ew_layer);
-      layerpostop.push_back(true);
+      layerpostop.push_back(enable_postops);
       if (comments) {
         std::cout << "Element wise (relu) added to layers" << '\n';
       }

app/Graph/build.hpp

@@ -61,7 +61,8 @@ void build_graph(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                  it_lab_ai::RuntimeOptions options, bool comments);
 void build_graph_linear(it_lab_ai::Graph& graph, it_lab_ai::Tensor& input,
                         it_lab_ai::Tensor& output,
-                        it_lab_ai::RuntimeOptions options, bool comments);
+                        it_lab_ai::RuntimeOptions options, bool comments,
+                        bool enable_postops = true);
 std::unordered_map<int, std::string> load_class_names(
     const std::string& filename);
 

app/Graph/onnx_subgraphs.cpp

@@ -1,26 +1,85 @@
 #include <algorithm>
 #include <filesystem>
 #include <iomanip>
+#include <iostream>
 #include <numeric>
+#include <opencv2/opencv.hpp>
 #include <sstream>
 #include <unordered_map>
 
 #include "build.hpp"
 #include "graph_transformations/graph_transformations.hpp"
+#include "layers_fused/ConvRelu.hpp"
 #include "perf/benchmarking.hpp"
 
 using namespace it_lab_ai;
 
+void alexnet_inf_careless(Graph& graph, const RuntimeOptions& options,
+                          Tensor& input, Tensor& output) {
+  auto* o = new Tensor(output);
+  auto* i = new Tensor(input);
+  graph.inference(options);
+  if (graph.getLayersCount() == 0) {
+    throw std::runtime_error("No layers");
+  }
+  graph.setOutput(graph.getLayerFromID(graph.getLayersCount() - 1), *o);
+  graph.setInput(graph.getLayerFromID(0), *i);
+}
+
+void alexnet_comparison() {
+  std::vector<size_t> counts = {979, 1134, 1031, 1009, 981,
+                                891, 957,  1027, 973,  1008};
+  size_t sum = std::accumulate(counts.begin(), counts.end(), size_t{0});
+  int count_pic = static_cast<int>(sum) + 10;
+  std::vector<float> res(count_pic * 28 * 28, 1.0F);
+  Tensor input;
+  Shape sh1({1, 5, 5, 3});
+  std::vector<float> vec;
+  vec.reserve(75);
+  for (int i = 0; i < 75; ++i) {
+    vec.push_back(3);
+  }
+  Tensor output = make_tensor(vec, sh1);
+
+  Shape sh({static_cast<size_t>(count_pic), 1, 28, 28});
+  Tensor t = make_tensor<float>(res, sh);
+  input = t;
+
+  RuntimeOptions options;
+  Graph graph;
+  Graph graph2;
+  build_graph_linear(graph, input, output, options, true, false);
+  Graph subgraph;
+  std::shared_ptr<Layer> layer_0 = std::make_shared<ConvolutionalLayer>();
+  std::shared_ptr<Layer> layer_1 = std::make_shared<EWLayer>("relu");
+  subgraph.setInput(layer_0, input);
+  subgraph.makeConnection(layer_0, layer_1);
+  std::shared_ptr<Layer> layer_to = std::make_shared<ConvReluLayer>(
+      std::dynamic_pointer_cast<ConvolutionalLayer>(layer_0));
+  changed_subgraphs(graph, subgraph, layer_to, graph2, input, options);
+  Tensor input_c = input;
+  Tensor output_c = output;
+  auto time1 = elapsed_time_avg<double, std::milli>(
+      4, alexnet_inf_careless, graph, options, input_c, output_c);
+  print_time_stats(graph);
+  auto time2 = elapsed_time_avg<double, std::milli>(
+      4, alexnet_inf_careless, graph2, options, input_c, output_c);
+  print_time_stats(graph2);
+  std::cout << time1 << " for unchanged graph\n";
+  std::cout << time2 << " for convrelu graph\n";
+}
+
 int main() {
   int type = 2;
   Tensor input = make_tensor(std::vector<int>({0}));
   RuntimeOptions options;
+  alexnet_comparison();
   if (type == 0) {
     Graph graph1;
     build_graph(graph1, input, input, MODEL_PATH_DENSENET_ONNX, options, false);
 
     Graph subgraph;
-    Tensor scale = make_tensor(std::vector<float>({1.0}));
+    Tensor scale = make_tensor(std::vector<float>({1.0F}));
     std::shared_ptr<Layer> layer_0 =
         std::make_shared<BatchNormalizationLayer>(scale, scale, scale, scale);
     std::shared_ptr<Layer> layer_1 = std::make_shared<EWLayer>("relu");

include/CMakeLists.txt

@@ -5,16 +5,17 @@ file(GLOB_RECURSE graphT_headers graph_transformations/*.h graph_transformations
 set(GRAPHT_HEADERS "${graphT_headers}" PARENT_SCOPE)
 
 file(GLOB_RECURSE layers_headers layers/*.h layers/*.hpp)
-set(LAYERS_HEADERS "${layers_headers}" PARENT_SCOPE)
+file(GLOB_RECURSE parallel_headers parallel/*.h parallel/*.hpp)
+set(LAYERS_HEADERS "${layers_headers}" "${parallel_headers}" PARENT_SCOPE)
 
 file(GLOB_RECURSE layers_oneDNN_headers layers_oneDNN/*.h layers_oneDNN/*.hpp)
 set(LAYERS_ONEDNN_HEADERS "${layers_oneDNN_headers}" PARENT_SCOPE)
 
+file(GLOB_RECURSE layers_fused_headers layers_fused/*.h layers_fused/*.hpp)
+set(LAYERS_FUSED_HEADERS "${layers_fused_headers}" PARENT_SCOPE)
+
 file(GLOB_RECURSE perf_headers perf/*.h perf/*.hpp)
 set(PERF_HEADERS "${perf_headers}" PARENT_SCOPE)
 
 file(GLOB_RECURSE reader_headers Weights_Reader/*.h Weights_Reader/*.hpp)
 set(READER_HEADERS "${reader_headers}" PARENT_SCOPE)
-
-file(GLOB_RECURSE parallel_headers parallel/*.h parallel/*.hpp)
-set(LAYERS_HEADERS "${parallel_headers}" PARENT_SCOPE)

include/graph/graph.hpp

@@ -31,7 +31,8 @@ static std::unordered_map<LayerType, std::string> label_map = {
     {kReshape, "Reshape"},
     {kSoftmax, "Softmax"},
     {kReduce, "Reduce"},
-    {kBatchNormalization, "Normalization"}};
+    {kBatchNormalization, "Normalization"},
+    {kConvRelu, "ConvRelu"}};
 
 struct LayerTimeStats {
   std::string layer_name;

include/graph_transformations/graph_transformations.hpp

@@ -2,8 +2,10 @@
 #include <vector>
 
 #include "graph/graph.hpp"
+#include "layers/ConvLayer.hpp"
 #include "layers/EWLayer.hpp"
 #include "layers/Layer.hpp"
+#include "layers_fused/ConvRelu.hpp"
 
 namespace it_lab_ai {
 

include/layers/ConvLayer.hpp

@@ -58,6 +58,19 @@ class ConvolutionalLayer : public Layer {
     dilations_ = dilations;
     useLegacyImpl_ = useLegacyImpl;
   }
+
+  [[nodiscard]] std::vector<size_t> getNumericParams() const {
+    std::vector<size_t> res = {stride_, pads_, dilations_, group_};
+    return res;
+  }
+
+  [[nodiscard]] std::vector<std::shared_ptr<Tensor>> getTensorParams() {
+    std::vector<std::shared_ptr<Tensor>> res = {kernel_, bias_};
+    return res;
+  }
+
+  [[nodiscard]] bool getLegacyImplBool() const { return useLegacyImpl_; }
+
   void run(const std::vector<Tensor>& input,
            std::vector<Tensor>& output) override;
   void run(const std::vector<Tensor>& input, std::vector<Tensor>& output,

include/layers/Layer.hpp

@@ -34,7 +34,8 @@ enum LayerType : uint8_t {
   kReshape,
   kSoftmax,
   kMatmul,
-  kBatchNormalization
+  kBatchNormalization,
+  kConvRelu
 };
 
 enum ImplType : uint8_t { kDefault, kTBB, kSTL };

include/layers/Tensor.hpp

@@ -170,7 +170,7 @@ std::vector<T>* Tensor::as() {
 }
 
 template <typename T>
-const std::vector<T>* Tensor::as() const {
+[[nodiscard]] const std::vector<T>* Tensor::as() const {
   if (GetTypeEnum<T>() != type_) {
     throw std::invalid_argument("Template type doesn't fit this Tensor");
   }