issue/1135 layernorm module

Author: PanZezhong1725

include/infinicore/nn/layernorm.hpp include/infinicore/nn/layer_norm.hppinclude/infinicore/nn/layernorm.hpp renamed to include/infinicore/nn/layer_norm.hpp

@@ -14,13 +14,11 @@ class LayerNorm : public Module {
      * @param eps Small constant for numerical stability (default: 1e-6)
      * @param dtype Data type for the weight (default: DataType::F32)
      * @param device Device to create the weight on
-     * @param elementwise_affine Whether to include learnable affine weight and bias parameters (default: true)
      */
     LayerNorm(size_t normalized_shape,
               double eps = 1e-6,
               const DataType &dtype = DataType::F32,
-              const Device &device = Device(),
-              bool elementwise_affine = true);
+              const Device &device = Device());
 
     /**
      * @brief Forward pass: apply LayerNorm
@@ -35,6 +33,8 @@ class LayerNorm : public Module {
      */
     Tensor forward(const Tensor &x) const;
 
+    // Module information
+    size_t normalized_shape() const { return normalized_shape_; }
     double eps() const { return eps_; }
     DataType dtype() const { return dtype_; }
 
@@ -54,6 +54,6 @@ class LayerNorm : public Module {
     size_t normalized_shape_; // Size of the feature dimension
     double eps_;              // Epsilon for numerical stability
     DataType dtype_;          // Data type for weight
-    bool elementwise_affine_; // Whether to use learnable affine parameters
 };
+
 } // namespace infinicore::nn

include/infinicore/ops.hpp

@@ -22,6 +22,7 @@
 #include "ops/hardswish.hpp"
 #include "ops/hardtanh.hpp"
 #include "ops/kv_caching.hpp"
+#include "ops/layer_norm.hpp"
 #include "ops/matmul.hpp"
 #include "ops/ones.hpp"
 #include "ops/paged_attention.hpp"

src/infinicore/nn/layer_norm.cc

@@ -0,0 +1,27 @@
+#include "infinicore/nn/layer_norm.hpp"
+#include "infinicore/ops.hpp"
+#include <cmath>
+#include <stdexcept>
+
+namespace infinicore::nn {
+
+LayerNorm::LayerNorm(size_t normalized_shape, double eps, const DataType &dtype, const Device &device)
+    : normalized_shape_(normalized_shape),
+      eps_(eps),
+      dtype_(dtype) {
+
+    device_ = device;
+
+    INFINICORE_NN_PARAMETER_INIT(weight, ({normalized_shape}, dtype_, device));
+    INFINICORE_NN_PARAMETER_INIT(bias, ({normalized_shape}, dtype_, device));
+}
+
+Tensor LayerNorm::forward(const Tensor &x) const {
+    return op::layer_norm(x, weight_, bias_, static_cast<float>(eps_));
+}
+
+std::string LayerNorm::extra_repr() const {
+    return "LayerNorm(normalized_shape=" + std::to_string(normalized_shape_) + ", eps=" + std::to_string(eps_) + ", dtype=" + std::to_string(static_cast<int>(dtype_)) + ")";
+}
+
+} // namespace infinicore::nn