Mul, Sub and Add Layer with broadcasting support (#186)

I will also add the functionality of element-by-element subtraction and
element-by-element addition layers - sub and add

Author: Semyon1104

app/Accuracy/accuracy_check.cpp

@@ -7,7 +7,7 @@
 #include "layers/OutputLayer.hpp"
 #include "layers/PoolingLayer.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main() {
   std::string image_path = IMAGE1_PATH;

app/AccuracyImgNet/accimgnet.cpp

@@ -10,7 +10,7 @@
 #include "layers/OutputLayer.hpp"
 #include "layers/PoolingLayer.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 bool cmp_by_first(const std::pair<size_t, std::string>& a,
                   const std::pair<size_t, std::string>& b) {

app/Graph/acc_check_mnist.cpp

@@ -5,7 +5,7 @@
 #include "build.cpp"
 #include "build.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   bool parallel = false;

app/Graph/build.cpp

@@ -86,7 +86,7 @@ void build_graph(Tensor& input, Tensor& output, bool comments,
       Tensor tmp_bias = make_tensor(tensor.get_bias());
       Tensor tmp_tensor =
           Tensor(Shape({tensor.get_shape()[1], tensor.get_shape()[0]}),
-                 itlab_2023::Type::kFloat);
+                 it_lab_ai::Type::kFloat);
       // kernel is always transposed ?
       for (size_t h = 0; h < tensor.get_shape()[0]; h++) {
         for (size_t w = 0; w < tensor.get_shape()[1]; w++) {

app/Graph/graph_build.cpp

@@ -2,7 +2,7 @@
 #include "build.hpp"
 
 namespace fs = std::filesystem;
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   std::string image_folder = IMAGE1_PATH;

include/Weights_Reader/reader_weights.hpp

@@ -5,7 +5,7 @@
 #include "layers/Tensor.hpp"
 
 using json = nlohmann::json;
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 json read_json(const std::string& filename);
 void extract_values_from_json(const json& j, std::vector<float>& values);

include/graph/graph.hpp

@@ -10,7 +10,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class Graph {
   int BiggestSize_;
@@ -148,4 +148,4 @@ class Graph {
   std::vector<Tensor> getWEIGHTS() { return weights_; }
 #endif
 };
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/BinaryOpLayer.hpp

@@ -0,0 +1,56 @@
+#pragma once
+#include <algorithm>
+#include <memory>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+#include "Tensor.hpp"
+#include "layers/Layer.hpp"
+
+namespace it_lab_ai {
+
+class BinaryOpLayer : public Layer {
+ public:
+  enum class Operation : uint8_t { kMul, kAdd, kSub, kDiv };
+
+  BinaryOpLayer() = default;
+  explicit BinaryOpLayer(Operation op) : op_(op) {}
+
+  static std::string get_name() { return "Binary Operation Layer"; }
+  void run(const Tensor& input, Tensor& output) override;
+  void run(const Tensor& A, const Tensor& B, Tensor& output);
+  static bool is_scalar_tensor(const Tensor& t);
+
+#ifdef ENABLE_STATISTIC_WEIGHTS
+  Tensor get_weights() override {
+    std::vector<int> v = {0};
+    return make_tensor(v);
+  }
+#endif
+
+ private:
+  Operation op_ = Operation::kMul;
+
+  template <typename ValueType>
+  void run_with_scalar_impl(const Tensor& input, ValueType scalar,
+                            Tensor& output) const;
+  template <typename ValueType>
+  void run_broadcast_impl(const Tensor& A, const Tensor& B, Tensor& output,
+                          const Shape& output_shape) const;
+  void run_with_scalar(const Tensor& input, float scalar, Tensor& output) const;
+
+  static bool can_broadcast(const Shape& shape_A, const Shape& shape_B);
+  static Shape calculate_broadcasted_shape(const Shape& shape_A,
+                                           const Shape& shape_B);
+  static std::vector<size_t> get_strides(const Shape& shape);
+  static size_t get_broadcasted_index(
+      size_t flat_index, const Shape& input_shape, const Shape& output_shape,
+      const std::vector<size_t>& input_strides,
+      const std::vector<size_t>& output_strides);
+
+  template <typename ValueType>
+  class BinaryOpLayerImpl;
+};
+
+}  // namespace it_lab_ai

include/layers/ConvLayer.hpp

@@ -6,7 +6,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class ConvolutionalLayer : public Layer {
  private:
@@ -402,4 +402,4 @@ void Conv4DSTL(const Tensor& input, const Tensor& kernel_, const Tensor& bias_,
   output = make_tensor<ValueType>(one_d_vector, sh);
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/DropOutLayer.hpp

@@ -3,7 +3,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class DropOutLayer : public Layer {
  private:
@@ -16,4 +16,4 @@ class DropOutLayer : public Layer {
   void run(const Tensor& input, Tensor& output) override;
 };
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai