fix: handle 1D tensors and default DenseLayer to identity

Author: ooples

src/NeuralNetworks/Layers/DenseLayer.cs

@@ -151,12 +151,12 @@ public class DenseLayer<T> : LayerBase<T>
     /// <para><b>For Beginners:</b> This tells you how many individual numbers the layer can adjust during training.
     /// 
     /// The parameter count:
-    /// - Equals (number of inputs × number of outputs) + number of outputs
+    /// - Equals (number of inputs � number of outputs) + number of outputs
     /// - First part counts the weights, second part counts the biases
     /// - Higher numbers mean more flexibility but also more risk of overfitting
     /// 
     /// For example, a dense layer with 100 inputs and 50 outputs would have
-    /// 100 × 50 = 5,000 weights plus 50 biases, for a total of 5,050 parameters.
+    /// 100 � 50 = 5,000 weights plus 50 biases, for a total of 5,050 parameters.
     /// </para>
     /// </remarks>
     public override int ParameterCount => (_weights.Rows * _weights.Columns) + _biases.Length;
@@ -190,7 +190,7 @@ public class DenseLayer<T> : LayerBase<T>
     /// </summary>
     /// <param name="inputSize">The number of input neurons.</param>
     /// <param name="outputSize">The number of output neurons.</param>
-    /// <param name="activationFunction">The activation function to apply. Defaults to ReLU if not specified.</param>
+    /// <param name="activationFunction">The activation function to apply. Defaults to Identity (linear) if not specified.</param>
     /// <remarks>
     /// <para>
     /// This constructor creates a dense layer with the specified number of input and output neurons.
@@ -205,14 +205,14 @@ public class DenseLayer<T> : LayerBase<T>
     /// - What mathematical function to apply to the results (activation)
     /// 
     /// For example, a layer with inputSize=784 and outputSize=10 could connect the flattened
-    /// pixels of a 28×28 image to 10 output neurons (one for each digit 0-9).
+    /// pixels of a 28�28 image to 10 output neurons (one for each digit 0-9).
     /// 
     /// The layer automatically initializes all the weights and biases with carefully chosen
     /// starting values that help with training.
     /// </para>
     /// </remarks>
     public DenseLayer(int inputSize, int outputSize, IActivationFunction<T>? activationFunction = null)
-        : base([inputSize], [outputSize], activationFunction ?? new ReLUActivation<T>())
+        : base([inputSize], [outputSize], activationFunction ?? new IdentityActivation<T>())
     {
         _weights = new Matrix<T>(outputSize, inputSize);
         _biases = new Vector<T>(outputSize);
@@ -226,7 +226,7 @@ public DenseLayer(int inputSize, int outputSize, IActivationFunction<T>? activat
     /// </summary>
     /// <param name="inputSize">The number of input neurons.</param>
     /// <param name="outputSize">The number of output neurons.</param>
-    /// <param name="vectorActivation">The vector activation function to apply. Defaults to ReLU if not specified.</param>
+    /// <param name="vectorActivation">The vector activation function to apply. Defaults to Identity (linear) if not specified.</param>
     /// <remarks>
     /// <para>
     /// This constructor creates a dense layer with the specified number of input and output neurons
@@ -247,7 +247,7 @@ public DenseLayer(int inputSize, int outputSize, IActivationFunction<T>? activat
     /// </para>
     /// </remarks>
     public DenseLayer(int inputSize, int outputSize, IVectorActivationFunction<T>? vectorActivation = null)
-        : base([inputSize], [outputSize], vectorActivation ?? new ReLUActivation<T>())
+        : base([inputSize], [outputSize], vectorActivation ?? new IdentityActivation<T>())
     {
         _weights = new Matrix<T>(outputSize, inputSize);
         _biases = new Vector<T>(outputSize);
@@ -368,18 +368,19 @@ public void SetWeights(Matrix<T> weights)
     public override Tensor<T> Forward(Tensor<T> input)
     {
         _lastInput = input;
-        int batchSize = input.Shape[0];
 
-        var flattenedInput = input.Reshape(batchSize, input.Shape[1]);
-        var output = flattenedInput.Multiply(_weights.Transpose()).Add(_biases);
+        var (input2D, squeezeOutput) = EnsureRank2BatchFirst(input);
+        var output = input2D.Multiply(_weights.Transpose()).Add(_biases);
 
         if (UsingVectorActivation)
         {
-            return VectorActivation!.Activate(output);
+            var activated = VectorActivation!.Activate(output);
+            return squeezeOutput ? activated.Reshape([activated.Shape[1]]) : activated;
         }
         else
         {
-            return ApplyActivation(output);
+            var activated = ApplyActivation(output);
+            return squeezeOutput ? activated.Reshape([activated.Shape[1]]) : activated;
         }
     }
 
@@ -413,33 +414,74 @@ public override Tensor<T> Backward(Tensor<T> outputGradient)
         if (_lastInput == null)
             throw new InvalidOperationException("Forward pass must be called before backward pass.");
 
-        int batchSize = _lastInput.Shape[0];
+        var (lastInput2D, _) = EnsureRank2BatchFirst(_lastInput);
+        int batchSize = lastInput2D.Shape[0];
+
+        var (outputGradient2D, _) = EnsureRank2BatchFirst(outputGradient);
+        if (outputGradient2D.Shape[0] != batchSize)
+        {
+            if (outputGradient.Length % batchSize != 0)
+            {
+                throw new ArgumentException(
+                    $"Output gradient length ({outputGradient.Length}) is not divisible by batch size ({batchSize}).",
+                    nameof(outputGradient));
+            }
+
+            outputGradient2D = outputGradient.Reshape([batchSize, outputGradient.Length / batchSize]);
+        }
 
         Tensor<T> activationGradient;
         if (UsingVectorActivation)
         {
-            activationGradient = VectorActivation!.Derivative(outputGradient);
+            activationGradient = VectorActivation!.Derivative(outputGradient2D);
         }
         else
         {
             // Apply scalar activation derivative element-wise
-            activationGradient = new Tensor<T>(outputGradient.Shape);
-            for (int i = 0; i < outputGradient.Length; i++)
+            activationGradient = new Tensor<T>(outputGradient2D.Shape);
+            for (int i = 0; i < outputGradient2D.Length; i++)
             {
-                activationGradient[i] = ScalarActivation!.Derivative(outputGradient[i]);
+                activationGradient[i] = ScalarActivation!.Derivative(outputGradient2D[i]);
             }
         }
 
-        var flattenedInput = _lastInput.Reshape(batchSize, _lastInput.Shape[1]);
-
-        _weightsGradient = activationGradient.Transpose([1, 0]).ToMatrix().Multiply(flattenedInput.ToMatrix());
-        _biasesGradient = activationGradient.Sum([0]).ToMatrix().ToColumnVector();
+        _weightsGradient = activationGradient.Transpose([1, 0]).ToMatrix().Multiply(lastInput2D.ToMatrix());
+        _biasesGradient = activationGradient.Sum([0]).ToVector();
 
         var inputGradient = activationGradient.Multiply(_weights);
 
         return inputGradient.Reshape(_lastInput.Shape);
     }
 
+    private static (Tensor<T> Tensor2D, bool SqueezeOutput) EnsureRank2BatchFirst(Tensor<T> tensor)
+    {
+        if (tensor.Shape.Length == 1)
+        {
+            return (tensor.Reshape([1, tensor.Shape[0]]), true);
+        }
+
+        int batchSize = tensor.Shape[0];
+        if (batchSize <= 0)
+        {
+            throw new ArgumentException($"Invalid batch size ({batchSize}).", nameof(tensor));
+        }
+
+        int features = tensor.Length / batchSize;
+        if (tensor.Length != batchSize * features)
+        {
+            throw new ArgumentException(
+                $"Tensor length ({tensor.Length}) is not compatible with batch size ({batchSize}).",
+                nameof(tensor));
+        }
+
+        if (tensor.Shape.Length == 2 && tensor.Shape[1] == features)
+        {
+            return (tensor, false);
+        }
+
+        return (tensor.Reshape([batchSize, features]), false);
+    }
+
     /// <summary>
     /// Updates the layer's parameters (weights and biases) using the calculated gradients.
     /// </summary>
@@ -641,4 +683,4 @@ public override LayerBase<T> Clone()
         copy.SetParameters(GetParameters());
         return copy;
     }
-}
+}