Sigmoid in EWLayer (#194)

Author: Semyon1104

include/layers/EWLayer.hpp

@@ -2,6 +2,7 @@
 #include <algorithm>
 #include <cmath>
 #include <string>
+#include <type_traits>
 #include <utility>
 
 #include "layers/Layer.hpp"
@@ -87,9 +88,26 @@ std::vector<ValueType> EWLayerImpl<ValueType>::run(
              static_cast<ValueType>(beta_);
     };
     std::transform(input.begin(), input.end(), res.begin(), linear);
+  } else if (func_ == "sigmoid") {
+    auto sigmoid = [](ValueType x) -> ValueType {
+      if constexpr (std::is_integral_v<ValueType>) {
+        auto x_float = static_cast<float>(x);
+        float result = 1.0F / (1.0F + std::exp(-x_float));
+        return static_cast<ValueType>(std::round(result));
+      } else {
+        if (x >= ValueType(0)) {
+          ValueType z = std::exp(-x);
+          return ValueType(1) / (ValueType(1) + z);
+        }
+        ValueType z = std::exp(x);
+        return z / (ValueType(1) + z);
+      }
+    };
+    std::transform(input.cbegin(), input.cend(), res.begin(), sigmoid);
   } else {
     throw std::invalid_argument("No such function for EWLayer");
   }
+
   return res;
 }
 

test/single_layer/test_ewlayer.cpp

@@ -54,7 +54,25 @@ INSTANTIATE_TEST_SUITE_P(
         std::make_tuple(basic_data2,
                         EWLayerImpl<double>({2, 2}, "linear", 2.0F, 1.0F),
                         std::vector<double>({3.0, -1.0, 5.0, -3.0}),
-                        std::function<double(double)>())));
+                        std::function<double(double)>()),
+        std::make_tuple(std::vector<double>({0.0, 1.0, -1.0}),
+                        EWLayerImpl<double>({3}, "sigmoid"),
+                        std::vector<double>(),
+                        std::function<double(double)>([](double x) {
+                          return 1.0 / (1.0 + std::exp(-x));
+                        })),
+        std::make_tuple(std::vector<double>{-100.0, -50.0, 0.0, 50.0, 100.0},
+                        EWLayerImpl<double>({5}, "sigmoid"),
+                        std::vector<double>(),
+                        std::function<double(double)>([](double x) {
+                          if (x >= 0) {
+                            double z = std::exp(-x);
+                            return 1.0 / (1.0 + z);
+                          } else {
+                            double z = std::exp(x);
+                            return z / (1.0 + z);
+                          }
+                        }))));
 
 TEST(ewlayer, new_ewlayer_can_relu_float) {
   EWLayer layer("relu");
@@ -100,3 +118,54 @@ TEST(ewlayer, new_ewlayer_throws_with_invalid_function) {
 TEST(ewlayer, get_layer_name) {
   EXPECT_EQ(EWLayer::get_name(), "Element-wise layer");
 }
+
+TEST(ewlayer, new_ewlayer_can_sigmoid_float) {
+  EWLayer layer("sigmoid");
+  Tensor input = make_tensor<float>({0.0F, -1.0F, 1.0F, 2.0F});
+  Tensor output;
+  std::vector<float> expected_output = {0.5F, 1.0F / (1.0F + std::exp(1.0F)),
+                                        1.0F / (1.0F + std::exp(-1.0F)),
+                                        1.0F / (1.0F + std::exp(-2.0F))};
+  layer.run(input, output);
+  for (size_t i = 0; i < 4; i++) {
+    EXPECT_NEAR((*output.as<float>())[i], expected_output[i], 1e-5F);
+  }
+}
+
+TEST(ewlayer, new_ewlayer_can_sigmoid_int) {
+  EWLayer layer("sigmoid");
+  Tensor input = make_tensor<int>({0, -100, 100, 1, -1});
+  Tensor output;
+  layer.run(input, output);
+
+  std::vector<int> expected = {1, 0, 1, 1, 0};
+  for (size_t i = 0; i < expected.size(); ++i) {
+    EXPECT_EQ((*output.as<int>())[i], expected[i]);
+  }
+}
+
+TEST(ewlayer, new_ewlayer_can_sigmoid_float_extreme_values) {
+  EWLayer layer("sigmoid");
+  Tensor input = make_tensor<float>({0.0F, -1.0F, 1.0F, 2.0F, -100.0F, 100.0F});
+  Tensor output;
+
+  auto stable_sigmoid = [](float x) {
+    if (x >= 0) {
+      float z = std::exp(-x);
+      return 1.0F / (1.0F + z);
+    } else {
+      float z = std::exp(x);
+      return z / (1.0F + z);
+    }
+  };
+
+  std::vector<float> expected_output = {
+      stable_sigmoid(0.0F), stable_sigmoid(-1.0F),   stable_sigmoid(1.0F),
+      stable_sigmoid(2.0F), stable_sigmoid(-100.0F), stable_sigmoid(100.0F)};
+
+  layer.run(input, output);
+
+  for (size_t i = 0; i < expected_output.size(); i++) {
+    EXPECT_NEAR((*output.as<float>())[i], expected_output[i], 1e-5F);
+  }
+}