Merge pull request #6 from techpro-studio/dev

Polished

Author: alexmay23

CMakeLists.txt

@@ -43,7 +43,7 @@ set(PRIVATE_HEADERS
     nntoolkitcore/core/ops.h
     nntoolkitcore/core/loop.h
     nntoolkitcore/core/memory.h
-        nntoolkitcore/layers/shared.h)
+    nntoolkitcore/layers/shared.h)
 
 set(SOURCES
     nntoolkitcore/layers/activation.c
@@ -63,6 +63,9 @@ set(SOURCES
     nntoolkitcore/layers/private/recurrent_private.h
     nntoolkitcore/core/memory.c
     nntoolkitcore/layers/bidirectional.c
+    nntoolkitcore/layers/private/weights_private.c
+    nntoolkitcore/layers/private/weights_private.h
+    nntoolkitcore/layers/recurrent.c
 )
 
 if(APPLE)

nntoolkitcore/core/debug.h

@@ -6,8 +6,8 @@
 //  Copyright © 2020 Alex. All rights reserved.
 //
 
-#ifndef degub_h
-#define degub_h
+#ifndef debug_h
+#define debug_h
 
 #include <stdio.h>
 #if defined __cplusplus
@@ -24,4 +24,4 @@ void print_tensor(const float *tensor, int *shape, int shapeSize);
 }
 #endif
 
-#endif /* degub_h */
+#endif /* debug_h */

nntoolkitcore/core/ops.h

@@ -9,12 +9,12 @@
 #ifndef ops_h
 #define ops_h
 
+#include "stdbool.h"
 
 #if defined __cplusplus
 extern "C" {
 #endif
 
-#include "stdbool.h"
 
 void op_vec_sub(const float *a, const float *b, float *result, int size);
 

nntoolkitcore/layers/batch_norm.c

@@ -97,10 +97,10 @@ void BatchNormDestroy(BatchNorm filter) {
 BatchNormGradient *BatchNormGradientCreate(BatchNormConfig config, BatchNormTrainingConfig training_config) {
     BatchNormGradient *grad = malloc(sizeof(BatchNormGradient));
     int feat = config.feature_channels;
-    int buff = 2 * feat * training_config.mini_batch_size + feat * config.count * training_config.mini_batch_size;
+    int buff = 2 * feat + feat * config.count * training_config.mini_batch_size;
     grad->d_beta = f_malloc(buff);
-    grad->d_gamma = grad->d_beta + feat * training_config.mini_batch_size;
-    grad->d_x = grad->d_gamma + feat * training_config.mini_batch_size;
+    grad->d_gamma = grad->d_beta + feat;
+    grad->d_x = grad->d_gamma + feat;
     return grad;
 }
 

nntoolkitcore/layers/conv_1d.c

@@ -10,10 +10,12 @@
 #include "nntoolkitcore/core/loop.h"
 #include "nntoolkitcore/core/ops.h"
 #include "nntoolkitcore/core/memory.h"
+#include "nntoolkitcore/layers/private/weights_private.h"
 
 typedef struct {
     ConvTrainingConfig config;
     float *input_transposed;
+    DefaultGradient **batch_gradients;
 } Conv1dTrainingData;
 
 typedef struct {
@@ -27,26 +29,43 @@ struct Conv1dStruct {
     Conv1dTrainingData *training_data;
 };
 
+ConvWeightsSize conv1d_weight_size_from_config(Conv1dConfig config) {
+    int w_size = config.kernel_size * config.input_feature_channels * config.output_feature_channels;
+    int sum = w_size + config.output_feature_channels;
+    return (DefaultWeightsSize) {.w = w_size, .b = config.output_feature_channels, .sum = sum};
+}
+
+
+static Conv1dInferenceData *conv1d_inference_data_create(Conv1dConfig config) {
+    Conv1dInferenceData *data = malloc(sizeof(Conv1dInferenceData));
+    data->buffer = malloc(config.input_size * config.input_feature_channels * sizeof(float));
+    return data;
+}
+
 static Conv1dTrainingData *conv1d_training_data_create(Conv1dConfig config, ConvTrainingConfig training_config) {
     Conv1dTrainingData *data = malloc(sizeof(Conv1dTrainingData));
+    int b = training_config.mini_batch_size;
     data->config = training_config;
     data->input_transposed = malloc(config.input_feature_channels * config.input_size
-                                    * training_config.mini_batch_size * sizeof(float));
+                                    * b * sizeof(float));
+    data->batch_gradients = malloc(b * sizeof(DefaultGradient *));
+    for (int i = 0; i < b; ++i) {
+        data->batch_gradients[i] = default_gradient_create(conv1d_weight_size_from_config(config), 0);
+    }
     return data;
 }
 
 static void conv_training_data_destroy(Conv1dTrainingData *training_data) {
+    for (int i = 0; i < training_data->config.mini_batch_size; ++i) {
+        default_gradient_destroy(training_data->batch_gradients[i]);
+    }
+    free(training_data->batch_gradients);
     free(training_data->input_transposed);
     free(training_data);
 }
 
-static Conv1dInferenceData *conv1d_inference_data_create(Conv1dConfig config) {
-    Conv1dInferenceData *data = malloc(sizeof(Conv1dInferenceData));
-    data->buffer = malloc(config.input_size * config.input_feature_channels * sizeof(float));
-    return data;
-}
 
-static void conv1d_inference_data_destroy(Conv1dInferenceData* data) {
+static void conv1d_inference_data_destroy(Conv1dInferenceData *data) {
     free(data->buffer);
     free(data);
 }
@@ -70,11 +89,7 @@ Conv1dConfig Conv1dConfigCreate(int input_feature_channels, int output_feature_c
 Conv1d conv1d_create(Conv1dConfig config) {
     Conv1d filter = malloc(sizeof(struct Conv1dStruct));
     filter->config = config;
-    filter->weights = malloc(sizeof(ConvWeights));
-    int W_size = config.kernel_size * config.input_feature_channels * config.output_feature_channels;
-    int weights_size = W_size + config.output_feature_channels;
-    filter->weights->W = f_malloc(weights_size);
-    filter->weights->b = filter->weights->W + W_size;
+    filter->weights = default_weights_create(conv1d_weight_size_from_config(config));
     filter->training_data = NULL;
     filter->inference_data = NULL;
     return filter;
@@ -140,19 +155,13 @@ int Conv1dApplyInference(Conv1d filter, const float *input, float *output) {
 }
 
 ConvGradient *Conv1dCreateGradient(Conv1dConfig config, ConvTrainingConfig training_config) {
-    ConvGradient *gradient = malloc(sizeof(ConvGradient));
-    int d_x_size = config.input_size * config.input_feature_channels * training_config.mini_batch_size;
-    int d_w_size = config.input_feature_channels * config.output_feature_channels * config.kernel_size * training_config.mini_batch_size;
-    int grad_size = d_x_size + d_w_size + config.output_feature_channels * training_config.mini_batch_size;
-    gradient->d_W = f_malloc(grad_size);
-    gradient->d_X = gradient->d_W + d_w_size;
-    gradient->d_b = gradient->d_X + d_x_size;
-    return gradient;
+    return default_gradient_create(conv1d_weight_size_from_config(config),
+                                   training_config.mini_batch_size *
+                                   config.input_size * config.input_feature_channels);
 }
 
 void ConvGradientDestroy(ConvGradient *gradient) {
-    free(gradient->d_W);
-    free(gradient);
+    default_gradient_destroy(gradient);
 }
 
 int Conv1dApplyTrainingBatch(Conv1d filter, const float *input, float *output) {
@@ -174,13 +183,6 @@ int Conv1dApplyTrainingBatch(Conv1d filter, const float *input, float *output) {
 }
 
 void Conv1dCalculateGradient(Conv1d filter, ConvGradient *gradient, const float *d_out) {
-    int db_size = filter->config.output_feature_channels *
-    filter->training_data->config.mini_batch_size;
-    for (int o = 0; o < filter->config.output_size; ++o){
-        op_vec_add(gradient->d_b, d_out + o * db_size, gradient->d_b, db_size);
-    }
-
-
     int k_size = filter->config.kernel_size;
     int batch = filter->training_data->config.mini_batch_size;
     int in_ftrs = filter->config.input_feature_channels;
@@ -199,6 +201,12 @@ void Conv1dCalculateGradient(Conv1d filter, ConvGradient *gradient, const float
     //  out_n  d4  d5  d6
 
     for (int b = 0; b < batch; ++b) {
+        //db
+        float *db_batched = filter->training_data->batch_gradients[b]->d_b;
+        for (int o = 0; o < filter->config.output_size; ++o){
+            op_vec_add(db_batched, d_out + o * out_ftrs +  b * out_size, db_batched, out_ftrs);
+        }
+
         for (int out_f = 0; out_f < out_ftrs; ++out_f) {
             for (int out_n = 0; out_n < filter->config.output_size; ++out_n) {
 
@@ -219,7 +227,7 @@ void Conv1dCalculateGradient(Conv1d filter, ConvGradient *gradient, const float
 
                     float d_kernel[k_size];
                     op_vec_mul_sc(row_ptr, d_o, d_kernel, k_size);
-                    float *d_W = gradient->d_W + W_size * b + weights_offset;
+                    float *d_W = filter->training_data->batch_gradients[b]->d_W + weights_offset;
                     op_vec_add(d_W, d_kernel, d_W, k_size);
 
                     // d_X;
@@ -233,6 +241,7 @@ void Conv1dCalculateGradient(Conv1d filter, ConvGradient *gradient, const float
         }
         op_mat_transp(d_x_transposed + b * inp_size, gradient->d_X + b * inp_size, filter->config.input_size, in_ftrs);
     }
+    default_gradient_sum(filter->training_data->batch_gradients, gradient, conv1d_weight_size_from_config(filter->config), batch);
 }
 
 

nntoolkitcore/layers/conv_1d.h

@@ -30,16 +30,9 @@ typedef struct {
 
 Conv1dConfig Conv1dConfigCreate(int input_feature_channels, int output_feature_channels, int kernel_size, int stride, int inputSize);
 
-typedef struct {
-    float *W;
-    float *b;
-} ConvWeights;
-
-typedef struct {
-    float *d_W;
-    float *d_b;
-    float *d_X;
-} ConvGradient;
+typedef DefaultWeights ConvWeights;
+typedef DefaultWeightsSize ConvWeightsSize;
+typedef DefaultGradient ConvGradient;
 
 typedef struct Conv1dStruct* Conv1d;
 

nntoolkitcore/layers/dense.c

@@ -10,29 +10,46 @@
 #include "nntoolkitcore/core/ops.h"
 #include "nntoolkitcore/core/memory.h"
 #include "nntoolkitcore/core/loop.h"
+#include "nntoolkitcore/layers/private/weights_private.h"
 
 typedef struct {
     DenseTrainingConfig config;
     float *x;
     float *z;
     float *a;
     float *dz;
+    DefaultGradient **batch_gradients;
 } DenseTrainingData;
 
+DenseWeightsSize dense_weight_size_from_config(DenseConfig config){
+    int w_size = config.input_size * config.output_size;
+    int sum = w_size + config.output_size;
+    return (DefaultWeightsSize) { .w = w_size, .b = config.output_size, .sum = sum };
+}
+
 DenseTrainingData *dense_training_data_create(DenseConfig config, DenseTrainingConfig training_config) {
     DenseTrainingData *data = malloc(sizeof(DenseTrainingData));
     data->config = training_config;
-    int x_size = config.input_size * training_config.mini_batch_size;
-    int z_size = config.output_size * training_config.mini_batch_size;
+    int b = training_config.mini_batch_size;
+    int x_size = config.input_size * b;
+    int z_size = config.output_size * b;
     int buff_size = x_size + 3 * z_size;
     data->x = f_malloc(buff_size);
     data->z = data->x + x_size;
     data->a = data->z + z_size;
     data->dz = data->a + z_size;
+    data->batch_gradients = malloc( b * sizeof(DefaultGradient*));
+    for (int i = 0; i < b; ++i){
+        data->batch_gradients[i] = default_gradient_create(dense_weight_size_from_config(config), 0);
+    }
     return data;
 }
 
 void dense_training_data_destroy(DenseTrainingData *data) {
+    for (int i = 0; i < data->config.mini_batch_size; ++i){
+        default_gradient_destroy(data->batch_gradients[i]);
+    }
+    free(data->batch_gradients);
     free(data->x);
     free(data);
 }
@@ -55,20 +72,18 @@ DenseConfig DenseConfigCreate(int input_size, int output_size, ActivationFunctio
     return config;
 }
 
+
+
 Dense DenseCreateForInference(DenseConfig config) {
     Dense filter = malloc(sizeof(struct DenseStruct));
     filter->config = config;
     filter->training_data = NULL;
-    filter->weights = malloc(sizeof(DenseWeights));
-    int weights_size = config.input_size * (config.output_size + 1);
-    filter->weights->W = f_malloc(weights_size);
-    filter->weights->b = filter->weights->W + config.input_size * config.output_size;
+    filter->weights = default_weights_create(dense_weight_size_from_config(config));
     return filter;
 }
 
 void DenseDestroy(Dense filter) {
-    free(filter->weights->W);
-    free(filter->weights);
+    default_weights_destroy(filter->weights);
     if (filter->training_data) {
         dense_training_data_destroy(filter->training_data);
     }
@@ -82,19 +97,22 @@ Dense DenseCreateForTraining(DenseConfig config, DenseTrainingConfig training_co
 }
 
 DenseGradient *DenseGradientCreate(DenseConfig config, DenseTrainingConfig training_config) {
-    DenseGradient *grad = malloc(sizeof(DenseGradient));
-    int d_w_size = config.input_size * config.output_size * training_config.mini_batch_size;
-    int d_x_size = config.input_size * training_config.mini_batch_size;
-    int grad_size = d_w_size + d_x_size + config.output_size * training_config.mini_batch_size;
-    grad->d_W = f_malloc(grad_size);
-    grad->d_X = grad->d_W + d_w_size;
-    grad->d_b = grad->d_X + d_x_size;
-    return grad;
+    return default_gradient_create(
+        dense_weight_size_from_config(config),
+        training_config.mini_batch_size * config.input_size
+    );
 }
 
+DenseGradient *DenseGradientCreateFromFilter(Dense dense) {
+    if (dense->training_data == NULL){
+        return NULL;
+    }
+    return DenseGradientCreate(dense->config, dense->training_data->config);
+}
+
+
 void DenseGradientDestroy(DenseGradient *gradient) {
-    free(gradient->d_W);
-    free(gradient);
+    default_gradient_destroy(gradient);
 }
 
 DenseConfig DenseGetConfig(Dense filter) {
@@ -146,7 +164,8 @@ int DenseApplyTrainingBatch(Dense filter, const float *input, float *output) {
 void DenseCalculateGradient(Dense filter, DenseGradient *gradient, float *d_out) {
     int out = filter->config.output_size;
     int in = filter->config.input_size;
-    P_LOOP_START(filter->training_data->config.mini_batch_size, b)
+    int batch = filter->training_data->config.mini_batch_size;
+    P_LOOP_START(batch, b)
                 // dz = d_out * d_activation ?? 1;
         float *dz = filter->training_data->dz + b * out;
         if (filter->config.activation) {
@@ -156,11 +175,13 @@ void DenseCalculateGradient(Dense filter, DenseGradient *gradient, float *d_out)
             f_copy(dz, d_out + b * out, out);
         }
         //db = dz;
-        f_copy(gradient->d_b + b * out, dz, out);
+        f_copy(filter->training_data->batch_gradients[b]->d_b, dz, out);
         // DW = dz * X;
-        op_mat_mul(filter->training_data->x + b * in, dz, gradient->d_W + b * in * out, in, out, 1);
+        op_mat_mul(filter->training_data->x + b * in, dz, filter->training_data->batch_gradients[b]->d_W, in, out, 1);
         // DX = dz * W;
         op_mat_mul(filter->weights->W, dz, gradient->d_X + b * in, in, 1, out);
     P_LOOP_END
+    default_gradient_sum(filter->training_data->batch_gradients, gradient, dense_weight_size_from_config(filter->config), batch);
 }
 
+

nntoolkitcore/layers/dense.h

@@ -13,20 +13,14 @@
 #include "activation.h"
 #include "shared.h"
 
+
 #if defined __cplusplus
 extern "C" {
 #endif
 
-typedef struct {
-    float *W;
-    float *b;
-} DenseWeights;
-
-typedef struct {
-    float *d_W;
-    float *d_b;
-    float *d_X;
-} DenseGradient;
+typedef DefaultWeights DenseWeights;
+typedef DefaultWeightsSize DenseWeightsSize;
+typedef DefaultGradient DenseGradient;
 
 typedef DefaultTrainingConfig DenseTrainingConfig;
 
@@ -42,6 +36,8 @@ void DenseGradientDestroy(DenseGradient *gradient);
 
 typedef struct DenseStruct* Dense;
 
+DenseGradient* DenseGradientCreateFromFilter(Dense dense);
+
 DenseWeights* DenseGetWeights(Dense filter);
 
 DenseConfig DenseConfigCreate(int input_size, int output_size, ActivationFunction activation);