alpha scaling parameter to GEMM-based linear operations

Author: baominghelly

include/infinicore/nn/linear.hpp

@@ -28,6 +28,8 @@ class BaseLinear : public Module {
     size_t out_features() const { return out_features_; }
     bool has_bias() const { return has_bias_; }
     DataType dtype() const { return dtype_; }
+    float alpha() const { return alpha_; }
+    void set_alpha(float alpha) { alpha_ = alpha; }
 
     // Accessors for parameters
     Tensor weight() const { return weight_; }
@@ -56,6 +58,7 @@ class BaseLinear : public Module {
     size_t out_features_;
     bool has_bias_;
     DataType dtype_;
+    float alpha_ = 1.0f;
     std::shared_ptr<infinicore::quantization::BaseQuantization> quantization_ = std::make_shared<infinicore::quantization::NoneQuantization>(nullptr);
 };
 

include/infinicore/ops/linear.hpp

@@ -5,8 +5,8 @@
 
 namespace infinicore::op {
 
-Tensor linear(Tensor input, Tensor weight, std::optional<Tensor> bias);
+Tensor linear(Tensor input, Tensor weight, std::optional<Tensor> bias, float alpha = 1.0f);
 
-void linear_(Tensor out, Tensor input, Tensor weight, std::optional<Tensor> bias);
+void linear_(Tensor out, Tensor input, Tensor weight, std::optional<Tensor> bias, float alpha = 1.0f);
 
 } // namespace infinicore::op

src/infinicore/nn/linear.cc

@@ -78,7 +78,7 @@ Tensor BaseLinear::compute_linear(Tensor &input) const {
         Tensor weight_tensor = static_cast<const Tensor &>(weight_);
         std::optional<Tensor> bias_opt = has_bias_ ? std::make_optional<Tensor>(static_cast<const Tensor &>(bias_)) : std::nullopt;
 
-        auto output = infinicore::op::linear(input_contiguous->contiguous(), weight_tensor->contiguous(), bias_opt);
+        auto output = infinicore::op::linear(input_contiguous->contiguous(), weight_tensor->contiguous(), bias_opt, alpha_);
         return output;
     }
     }

src/infinicore/ops/linear/linear.cc

@@ -6,7 +6,8 @@ namespace infinicore::op {
 
 Tensor linear(Tensor input,
               Tensor weight,
-              std::optional<Tensor> bias) {
+              std::optional<Tensor> bias,
+              float alpha) {
 
     Size ndim = input->ndim();
     Size out_features = weight->shape()[0];
@@ -17,14 +18,15 @@ Tensor linear(Tensor input,
     auto out = Tensor::empty(output_shape, input->dtype(), input->device());
 
     // Inplace Calculate
-    linear_(out, input, weight, bias);
+    linear_(out, input, weight, bias, alpha);
     return out;
 }
 
 void linear_(Tensor out,
              Tensor input,
              Tensor weight,
-             std::optional<Tensor> bias) {
+             std::optional<Tensor> bias,
+             float alpha) {
 
     auto weight_shape = weight->shape();
     Size out_features = weight_shape[0];
@@ -43,7 +45,6 @@ void linear_(Tensor out,
     // linear transformation
     Tensor out_view = out->view({N, out_features});
     // Add bias
-    float alpha = 1.0f;
     float beta = 0.0f;
     if (bias.has_value()) {
         rearrange_(out_view,