ooples
diff --git a/‎Directory.Packages.props‎
Lines changed: 11 additions & 5 deletions b/‎Directory.Packages.props‎
Lines changed: 11 additions & 5 deletions
diff --git a/‎src/Models/Options/NBEATSModelOptions.cs‎
Lines changed: 20 additions & 0 deletions b/‎src/Models/Options/NBEATSModelOptions.cs‎
Lines changed: 20 additions & 0 deletions
diff --git a/‎src/TimeSeries/AutoformerModel.cs‎
Lines changed: 343 additions & 1732 deletions b/‎src/TimeSeries/AutoformerModel.cs‎
Lines changed: 343 additions & 1732 deletions
diff --git a/‎src/TimeSeries/DeepARModel.cs‎
Lines changed: 599 additions & 572 deletions b/‎src/TimeSeries/DeepARModel.cs‎
Lines changed: 599 additions & 572 deletions
diff --git a/‎src/TimeSeries/InformerModel.cs‎
Lines changed: 465 additions & 1187 deletions b/‎src/TimeSeries/InformerModel.cs‎
Lines changed: 465 additions & 1187 deletions
diff --git a/‎src/TimeSeries/NBEATSModel.cs‎
Lines changed: 392 additions & 5 deletions b/‎src/TimeSeries/NBEATSModel.cs‎
Lines changed: 392 additions & 5 deletions
diff --git a/‎src/TimeSeries/NHiTSModel.cs‎
Lines changed: 495 additions & 287 deletions b/‎src/TimeSeries/NHiTSModel.cs‎
Lines changed: 495 additions & 287 deletions
diff --git a/‎src/TimeSeries/TFT/GatedResidualNetwork.cs‎
Lines changed: 65 additions & 142 deletions b/‎src/TimeSeries/TFT/GatedResidualNetwork.cs‎
Lines changed: 65 additions & 142 deletions
@@ -229,11 +229,17 @@
          activations from a prior training step on the same thread) leaked straight into the concat
          result — an intermittent, pool-state-dependent NaN, exactly the failure mode behind the
          model-family shard flakes this PR targets. 0.111.2 is PUBLISHED to NuGet and is a superset of
-         0.111.1, so all rationale above is retained. -->
-    <PackageVersion Include="AiDotNet.Tensors" Version="0.111.2" />
-    <PackageVersion Include="AiDotNet.Native.OneDNN" Version="0.111.2" />
-    <PackageVersion Include="AiDotNet.Native.OpenBLAS" Version="0.111.2" />
-    <PackageVersion Include="AiDotNet.Native.CLBlast" Version="0.111.2" />
+         0.111.1, so all rationale above is retained.
+
+         Bumped 0.111.2 -> 0.112.0: brings AiDotNet.Tensors #759 (N-BEATS resident blow-up — zero only
+         accumulating multi-consumer grad buffers) and #764 (GPU-resident-parameter fused-Adam mistrain
+         fix), which together make this PR's TimeSeries GPU-resident training path train faithfully (the
+         resident run now lowers the eager-forward training MSE and improves held-out MSE, so the N-BEATS
+         correctness gate accepts it). 0.112.0 is PUBLISHED to NuGet and is a superset of 0.111.2. -->
+    <PackageVersion Include="AiDotNet.Tensors" Version="0.112.0" />
+    <PackageVersion Include="AiDotNet.Native.OneDNN" Version="0.112.0" />
+    <PackageVersion Include="AiDotNet.Native.OpenBLAS" Version="0.112.0" />
+    <PackageVersion Include="AiDotNet.Native.CLBlast" Version="0.112.0" />
     <!-- Microsoft / .NET -->
     <PackageVersion Include="Microsoft.CSharp" Version="4.7.0" />
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="10.0.9" />
 
@@ -53,6 +53,7 @@ public NBEATSModelOptions(NBEATSModelOptions<T> other)
         HiddenLayerSize = other.HiddenLayerSize;
         NumHiddenLayers = other.NumHiddenLayers;
         LearningRate = other.LearningRate;
+        GradientClipNorm = other.GradientClipNorm;
         Epochs = other.Epochs;
         BatchSize = other.BatchSize;
         ShareWeightsInStack = other.ShareWeightsInStack;
@@ -193,6 +194,25 @@ public NBEATSModelOptions(NBEATSModelOptions<T> other)
     /// </remarks>
     public double LearningRate { get; set; } = 0.001;
 
+    /// <summary>
+    /// Gets or sets the global-norm gradient-clipping threshold used during training. Set to 0 to disable.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// Before each optimizer step, the L2 norm of all parameter gradients is computed; if it exceeds this
+    /// threshold every gradient is scaled down so the norm equals it. Oreshkin et al. (2020) train N-BEATS with
+    /// gradient clipping — the deep doubly-residual stack otherwise explodes on fat-tailed real series at the
+    /// default learning rate: the normalized training loss DIVERGES upward (e.g. 1.20 → 1.55 over 15 epochs on
+    /// SPY daily returns) and inference predictions blow up to hundreds of times the target scale. Clipping keeps
+    /// the step bounded and makes training robust to outlier batches without lowering the learning rate.
+    /// </para>
+    /// <para><b>For Beginners:</b> Occasionally a training batch produces a huge gradient (from an outlier in the
+    /// data) that would send the model flying off course. Gradient clipping caps how big any single update can be,
+    /// like a speed limit — it keeps learning stable. 1.0 is a standard, safe default.
+    /// </para>
+    /// </remarks>
+    public double GradientClipNorm { get; set; } = 1.0;
+
     /// <summary>
     /// Gets or sets the number of training epochs.
     /// </summary>
 
@@ -5,212 +5,125 @@ namespace AiDotNet.TimeSeries.TFT;
 
 /// <summary>
 /// Gated Residual Network (GRN) as described in Lim et al. (2021).
-/// GRN(a, c) = LayerNorm(a + GLU(η₁)) where η₁ = W₁·η₂ + b₁,
-/// η₂ = ELU(W₂·a + W₃·c + b₂), and GLU(γ) = σ(W₄·γ + b₄) ⊙ (W₅·γ + b₅).
+/// GRN(a) = LayerNorm(a + GLU(η₁)) where η₁ = W₁·η₂ + b₁,
+/// η₂ = ELU(W₂·a + b₂), and GLU(γ) = σ(W₄·γ + b₄) ⊙ (W₅·γ + b₅).
 /// </summary>
+/// <remarks>
+/// This implementation is expressed entirely with batched <c>Engine.Tensor*</c> ops so a
+/// <see cref="Tensors.Engines.Autodiff.GradientTape{T}"/> differentiates it automatically
+/// (no hand-derived gradients) and every op is GPU-dispatchable. All layers operate on a
+/// flattened <c>[N, d]</c> matrix where each of the N rows is processed independently
+/// (token-wise), matching the Temporal Fusion Transformer tape-training campaign.
+/// The final layer normalization uses learned affine parameters (γ, β).
+/// </remarks>
 internal class GatedResidualNetwork<T>
 {
-    private static readonly INumericOperations<T> NumOps = MathHelper.GetNumericOperations<T>();
     private static IEngine Engine => AiDotNetEngine.Current;
 
     private readonly int _inputSize;
     private readonly int _hiddenSize;
     private readonly int _outputSize;
-    private readonly int _contextSize;
-    private readonly bool _hasContext;
 
-    // η₂ = ELU(W₂·a + W₃·c + b₂)
-    private Tensor<T> _w2; // [hiddenSize, inputSize]
-    private Tensor<T> _b2; // [hiddenSize]
-    private Tensor<T>? _w3; // [hiddenSize, contextSize] — only if context is used
+    // η₂ = ELU(W₂·a + b₂)
+    private readonly Tensor<T> _w2; // [hiddenSize, inputSize]
+    private readonly Tensor<T> _b2; // [hiddenSize]
 
     // η₁ = W₁·η₂ + b₁
-    private Tensor<T> _w1; // [outputSize, hiddenSize]
-    private Tensor<T> _b1; // [outputSize]
+    private readonly Tensor<T> _w1; // [outputSize, hiddenSize]
+    private readonly Tensor<T> _b1; // [outputSize]
 
     // GLU: σ(W₄·γ + b₄) ⊙ (W₅·γ + b₅)
-    private Tensor<T> _w4; // [outputSize, outputSize]
-    private Tensor<T> _b4; // [outputSize]
-    private Tensor<T> _w5; // [outputSize, outputSize]
-    private Tensor<T> _b5; // [outputSize]
+    private readonly Tensor<T> _w4; // [outputSize, outputSize]
+    private readonly Tensor<T> _b4; // [outputSize]
+    private readonly Tensor<T> _w5; // [outputSize, outputSize]
+    private readonly Tensor<T> _b5; // [outputSize]
 
-    // Skip connection projection (when inputSize != outputSize)
-    private Tensor<T>? _skipProjection; // [outputSize, inputSize]
+    // Skip connection projection (only when inputSize != outputSize)
+    private readonly Tensor<T>? _skipProjection; // [outputSize, inputSize]
 
-    public GatedResidualNetwork(int inputSize, int hiddenSize, int outputSize, int contextSize = 0, int? seed = null)
+    // Learned LayerNorm affine parameters
+    private readonly Tensor<T> _lnGamma; // [outputSize]
+    private readonly Tensor<T> _lnBeta;  // [outputSize]
+
+    public GatedResidualNetwork(int inputSize, int hiddenSize, int outputSize, int? seed = null)
     {
         _inputSize = inputSize;
         _hiddenSize = hiddenSize;
         _outputSize = outputSize;
-        _contextSize = contextSize;
-        _hasContext = contextSize > 0;
 
         var random = seed.HasValue ? new Random(seed.Value) : RandomHelper.CreateSeededRandom(42);
 
-        // He initialization for ELU
-        double std2 = Math.Sqrt(2.0 / _inputSize);
-        _w2 = CreateRandomTensor([_hiddenSize, _inputSize], std2, random);
+        // He initialization for the ELU layer
+        _w2 = CreateRandomTensor([_hiddenSize, _inputSize], Math.Sqrt(2.0 / _inputSize), random);
         _b2 = new Tensor<T>([_hiddenSize]);
 
-        if (_hasContext)
-        {
-            double std3 = Math.Sqrt(2.0 / _contextSize);
-            _w3 = CreateRandomTensor([_hiddenSize, _contextSize], std3, random);
-        }
-
-        double std1 = Math.Sqrt(2.0 / _hiddenSize);
-        _w1 = CreateRandomTensor([_outputSize, _hiddenSize], std1, random);
+        _w1 = CreateRandomTensor([_outputSize, _hiddenSize], Math.Sqrt(2.0 / _hiddenSize), random);
         _b1 = new Tensor<T>([_outputSize]);
 
-        // GLU weights — initialize small so gate starts near 0.5 (neutral)
+        // GLU weights — small so the gate starts near 0.5 (neutral)
         double stdGlu = Math.Sqrt(1.0 / _outputSize);
         _w4 = CreateRandomTensor([_outputSize, _outputSize], stdGlu, random);
         _b4 = new Tensor<T>([_outputSize]);
         _w5 = CreateRandomTensor([_outputSize, _outputSize], stdGlu, random);
         _b5 = new Tensor<T>([_outputSize]);
 
-        // Skip projection if dimensions don't match
         if (_inputSize != _outputSize)
         {
             _skipProjection = CreateRandomTensor([_outputSize, _inputSize], Math.Sqrt(1.0 / _inputSize), random);
         }
+
+        _lnGamma = CreateConstantTensor([_outputSize], 1.0);
+        _lnBeta = new Tensor<T>([_outputSize]);
     }
 
     /// <summary>
-    /// Forward pass: GRN(a, c) = LayerNorm(a + GLU(η₁))
+    /// Forward pass GRN(a) = LayerNorm(a + GLU(η₁)) on a flattened <c>[N, inputSize]</c> batch.
+    /// Returns <c>[N, outputSize]</c>. Fully tape-differentiable.
     /// </summary>
-    /// <param name="input">Primary input [batch, inputSize] or [inputSize]</param>
-    /// <param name="context">Optional static context [batch, contextSize] or [contextSize]</param>
-    public Tensor<T> Forward(Tensor<T> input, Tensor<T>? context = null)
+    public Tensor<T> Forward(Tensor<T> input)
     {
-        // η₂ = ELU(W₂·a + W₃·c + b₂)
-        var eta2 = LinearProject(input, _w2, _b2);
-
-        if (_hasContext && context != null && _w3 != null)
-        {
-            var contextProj = LinearProject(context, _w3, null);
-            eta2 = Engine.TensorAdd(eta2, contextProj);
-        }
-
-        eta2 = ApplyELU(eta2);
+        // η₂ = ELU(W₂·a + b₂)
+        var eta2 = Engine.ELU(Linear(input, _w2, _b2), 1.0);
 
         // η₁ = W₁·η₂ + b₁
-        var eta1 = LinearProject(eta2, _w1, _b1);
+        var eta1 = Linear(eta2, _w1, _b1);
 
         // GLU(η₁) = σ(W₄·η₁ + b₄) ⊙ (W₅·η₁ + b₅)
-        var gate = LinearProject(eta1, _w4, _b4);
-        gate = ApplySigmoid(gate);
-        var value = LinearProject(eta1, _w5, _b5);
+        var gate = Engine.Sigmoid(Linear(eta1, _w4, _b4));
+        var value = Linear(eta1, _w5, _b5);
         var gluOutput = Engine.TensorMultiply(gate, value);
 
-        // Skip connection: a (or projected a if sizes differ)
-        var skip = (_skipProjection != null)
-            ? LinearProject(input, _skipProjection, null)
-            : input;
-
-        // Residual + LayerNorm
-        var output = Engine.TensorAdd(skip, gluOutput);
-        output = ApplyLayerNorm(output);
+        // Skip connection (projected when input/output dims differ)
+        var skip = _skipProjection != null ? Linear(input, _skipProjection, null) : input;
 
-        return output;
+        // Residual + LayerNorm(γ, β)
+        var residual = Engine.TensorAdd(skip, gluOutput);
+        return Engine.LayerNorm(residual, _lnGamma, _lnBeta, 1e-5, out _, out _);
     }
 
-    /// <summary>
-    /// Collects all trainable parameters for tape-based autodiff.
-    /// </summary>
+    /// <summary>Collects all trainable parameter tensors for tape-based autodiff.</summary>
     public IEnumerable<Tensor<T>> GetTrainableParameters()
     {
         yield return _w2;
         yield return _b2;
-        if (_w3 != null) yield return _w3;
         yield return _w1;
         yield return _b1;
         yield return _w4;
         yield return _b4;
         yield return _w5;
         yield return _b5;
         if (_skipProjection != null) yield return _skipProjection;
+        yield return _lnGamma;
+        yield return _lnBeta;
     }
 
-    private static Tensor<T> LinearProject(Tensor<T> input, Tensor<T> weight, Tensor<T>? bias)
+    // Row-wise affine map: x[N, in] · W[out, in]^T + b[out] -> [N, out].
+    private static Tensor<T> Linear(Tensor<T> x, Tensor<T> weight, Tensor<T>? bias)
     {
-        // input: [N] or [batch, N], weight: [outSize, inSize]
         int outSize = weight.Shape[0];
-        int inSize = weight.Shape[1];
-
-        Tensor<T> result;
-        if (input.Shape.Length == 1 || (input.Shape.Length == 2 && input.Shape[0] == 1))
-        {
-            // Single vector: weight @ input
-            var inputCol = input.Reshape(inSize, 1);
-            result = Engine.TensorMatMul(weight, inputCol).Reshape(outSize);
-        }
-        else
-        {
-            // Batched: input @ weight^T
-            var weightT = weight.Transpose(new[] { 1, 0 });
-            result = Engine.TensorMatMul(input, weightT);
-        }
-
+        var result = Engine.TensorMatMul(x, Engine.TensorTranspose(weight));
         if (bias != null)
-        {
-            result = Engine.TensorAdd(result, bias.Reshape(result._shape));
-        }
-
-        return result;
-    }
-
-    private static Tensor<T> ApplyELU(Tensor<T> x)
-    {
-        var result = new Tensor<T>(x._shape);
-        var xSpan = x.Data.Span;
-        var rSpan = result.AsWritableSpan();
-        for (int i = 0; i < xSpan.Length; i++)
-        {
-            double val = NumOps.ToDouble(xSpan[i]);
-            rSpan[i] = NumOps.FromDouble(val >= 0 ? val : Math.Exp(val) - 1.0);
-        }
-        return result;
-    }
-
-    private static Tensor<T> ApplySigmoid(Tensor<T> x)
-    {
-        var result = new Tensor<T>(x._shape);
-        var xSpan = x.Data.Span;
-        var rSpan = result.AsWritableSpan();
-        for (int i = 0; i < xSpan.Length; i++)
-        {
-            double val = NumOps.ToDouble(xSpan[i]);
-            rSpan[i] = NumOps.FromDouble(1.0 / (1.0 + Math.Exp(-val)));
-        }
-        return result;
-    }
-
-    private static Tensor<T> ApplyLayerNorm(Tensor<T> x)
-    {
-        // Simple layer normalization over the last dimension
-        var span = x.Data.Span;
-        int len = span.Length;
-        if (len == 0) return x;
-
-        double sum = 0, sumSq = 0;
-        for (int i = 0; i < len; i++)
-        {
-            double v = NumOps.ToDouble(span[i]);
-            sum += v;
-            sumSq += v * v;
-        }
-
-        double mean = sum / len;
-        double variance = sumSq / len - mean * mean;
-        double invStd = 1.0 / Math.Sqrt(variance + 1e-5);
-
-        var result = new Tensor<T>(x._shape);
-        var rSpan = result.AsWritableSpan();
-        for (int i = 0; i < len; i++)
-        {
-            rSpan[i] = NumOps.FromDouble((NumOps.ToDouble(span[i]) - mean) * invStd);
-        }
+            result = Engine.TensorBroadcastAdd(result, Engine.Reshape(bias, [1, outSize]));
         return result;
     }
 
@@ -219,14 +132,24 @@ private static Tensor<T> CreateRandomTensor(int[] shape, double stddev, Random r
         int size = 1;
         foreach (var s in shape) size *= s;
         var data = new T[size];
+        var numOps = MathHelper.GetNumericOperations<T>();
         for (int i = 0; i < size; i++)
         {
-            // Normal distribution via Box-Muller
             double u1 = 1.0 - random.NextDouble();
             double u2 = random.NextDouble();
             double normal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Cos(2.0 * Math.PI * u2);
-            data[i] = NumOps.FromDouble(normal * stddev);
+            data[i] = numOps.FromDouble(normal * stddev);
         }
         return new Tensor<T>(shape, new Vector<T>(data));
     }
+
+    private static Tensor<T> CreateConstantTensor(int[] shape, double value)
+    {
+        int size = 1;
+        foreach (var s in shape) size *= s;
+        var data = new T[size];
+        var numOps = MathHelper.GetNumericOperations<T>();
+        for (int i = 0; i < size; i++) data[i] = numOps.FromDouble(value);
+        return new Tensor<T>(shape, new Vector<T>(data));
+    }
 }