diff --git a/src/Autodiff/GradientCheckpointing.cs b/src/Autodiff/GradientCheckpointing.cs deleted file mode 100644 index 339fa47ab2..0000000000 --- a/src/Autodiff/GradientCheckpointing.cs +++ /dev/null @@ -1,371 +0,0 @@ -using AiDotNet.Tensors.Engines.Autodiff; - -namespace AiDotNet.Autodiff; - -/// -/// Provides gradient checkpointing functionality for memory-efficient training. -/// -/// -/// -/// Gradient checkpointing (also known as activation checkpointing or memory checkpointing) -/// is a technique that trades computation time for memory by not storing all intermediate -/// activations during the forward pass. Instead, it recomputes them during the backward pass. -/// -/// For Beginners: When training large neural networks, storing all intermediate -/// results (activations) can use a lot of memory. Gradient checkpointing saves memory by: -/// -/// 1. Only storing activations at certain "checkpoints" -/// 2. During backpropagation, recomputing the activations between checkpoints -/// -/// This uses less memory but takes more time (roughly 30% more computation). -/// It's essential for training very large models that wouldn't otherwise fit in GPU memory. -/// -/// -/// This implementation follows patterns from PyTorch's torch.utils.checkpoint and -/// TensorFlow's tf.recompute_grad. -/// -/// -public static class GradientCheckpointing -{ - /// - /// Thread-local stack to track checkpoint boundaries during forward/backward passes. - /// - [ThreadStatic] - private static Stack>? _checkpointStack; - - /// - /// Executes a function with gradient checkpointing. - /// - /// The function to execute with checkpointing. - /// The input nodes to the function. - /// The output node from the function. - /// - /// - /// The function will be executed during the forward pass, but its intermediate - /// activations will not be saved. During the backward pass, the function will - /// be re-executed to recompute the needed activations. - /// - /// For Beginners: Wrap parts of your model in this function to save memory: - /// - /// - /// // Without checkpointing (uses more memory): - /// var output = layer1.Forward(input); - /// output = layer2.Forward(output); - /// - /// // With checkpointing (uses less memory): - /// var output = GradientCheckpointing<float>.Checkpoint( - /// () => { - /// var x = layer1.Forward(input); - /// return layer2.Forward(x); - /// }, - /// new[] { input } - /// ); - /// - /// - /// - public static ComputationNode Checkpoint( - Func> function, - IEnumerable> inputs) - { - var inputList = inputs.ToList(); - - // Create checkpoint context - var context = new CheckpointContext - { - Function = function, - Inputs = inputList, - SavedTensors = new Dictionary, Tensor>() - }; - - // Push context onto stack - if (_checkpointStack == null) - { - _checkpointStack = new Stack>(); - } - _checkpointStack.Push(context); - - // Suspend tape recording during checkpoint forward pass by using NoGradScope - // (ops within NoGradScope are not recorded to the active tape) - ComputationNode output; - using (new NoGradScope()) - { - output = function(); - } - - // Save only the inputs and output for recomputation - foreach (var input in inputList) - { - if (input.Value != null) - { - context.SavedTensors[input] = input.Value.Clone(); - } - } - context.Output = output; - context.OutputValue = output.Value?.Clone(); - - try - { - // With tape-based autodiff, the forward pass ran inside NoGradScope so ops - // weren't recorded. The saved tensors allow recomputation during backward. - // Return the output node directly — the tape handles gradient flow. - return output; - } - finally - { - // Pop context - _checkpointStack.Pop(); - } - } - - /// - /// Executes a function with gradient checkpointing, supporting multiple outputs. - /// - /// The function to execute with checkpointing. - /// The input nodes to the function. - /// The output nodes from the function. - public static IReadOnlyList> CheckpointMultiOutput( - Func>> function, - IEnumerable> inputs) - { - var inputList = inputs.ToList(); - - // Create checkpoint context - var context = new CheckpointContext - { - Inputs = inputList, - SavedTensors = new Dictionary, Tensor>() - }; - - if (_checkpointStack == null) - { - _checkpointStack = new Stack>(); - } - _checkpointStack.Push(context); - - IReadOnlyList> outputs; - using (new NoGradScope()) - { - outputs = function(); - } - - try - { - foreach (var input in inputList) - { - if (input.Value != null) - { - context.SavedTensors[input] = input.Value.Clone(); - } - } - - context.MultiOutputs = outputs.ToList(); - - // With tape-based autodiff, return the outputs directly. - return outputs; - } - finally - { - _checkpointStack.Pop(); - } - } - - - /// - /// Creates a sequential checkpoint that divides a sequence of layers into segments. - /// - /// The sequence of layer functions to checkpoint. - /// The input to the first layer. - /// Number of layers per checkpoint segment. Default: 2 - /// The output from the final layer. - /// - /// - /// This is a convenience method for checkpointing sequential models. It automatically - /// divides the layers into segments and applies checkpointing to each segment. - /// - /// For Beginners: For models with many sequential layers (like ResNet or Transformers), - /// this automatically applies checkpointing efficiently: - /// - /// - /// var layers = new List<Func<ComputationNode<float>, ComputationNode<float>>> - /// { - /// x => layer1.Forward(x), - /// x => layer2.Forward(x), - /// x => layer3.Forward(x), - /// x => layer4.Forward(x) - /// }; - /// - /// // Checkpoint every 2 layers - /// var output = GradientCheckpointing<float>.SequentialCheckpoint(layers, input, segmentSize: 2); - /// - /// - /// - public static ComputationNode SequentialCheckpoint( - IReadOnlyList, ComputationNode>> layers, - ComputationNode input, - int segmentSize = 2) - { - if (layers == null || layers.Count == 0) - { - return input; - } - - if (segmentSize <= 0) - { - throw new ArgumentOutOfRangeException(nameof(segmentSize), - segmentSize, "Segment size must be a positive integer."); - } - - var current = input; - int numSegments = (layers.Count + segmentSize - 1) / segmentSize; - - for (int seg = 0; seg < numSegments; seg++) - { - int startIdx = seg * segmentSize; - int endIdx = Math.Min(startIdx + segmentSize, layers.Count); - - var segmentLayers = layers.Skip(startIdx).Take(endIdx - startIdx).ToList(); - var segmentInput = current; - - current = Checkpoint( - () => - { - var x = segmentInput; - foreach (var layer in segmentLayers) - { - x = layer(x); - } - return x; - }, - new[] { segmentInput } - ); - } - - return current; - } - - /// - /// Estimates memory savings from using gradient checkpointing. - /// - /// Number of layers in the model. - /// Size of activations per layer in bytes. - /// Number of layers per checkpoint segment. - /// A tuple of (memory without checkpointing, memory with checkpointing, savings percentage). - /// - /// For Beginners: This helps you estimate how much memory you'll save: - /// - /// - /// var (without, with, savings) = GradientCheckpointing<float>.EstimateMemorySavings( - /// numLayers: 24, - /// activationSize: 100_000_000, // 100MB per layer - /// segmentSize: 4 - /// ); - /// // Result is available in the returned value - /// - /// - /// - public static (long WithoutCheckpoint, long WithCheckpoint, double SavingsPercent) EstimateMemorySavings( - int numLayers, - long activationSize, - int segmentSize = 2) - { - // Without checkpointing: store all activations - long withoutCheckpoint = numLayers * activationSize; - - // With checkpointing: store only sqrt(n) activations plus segment activations - int numSegments = (numLayers + segmentSize - 1) / segmentSize; - // Peak memory is: segment activations + checkpoint storage - long withCheckpoint = (segmentSize * activationSize) + (numSegments * activationSize); - - double savings = 1.0 - (double)withCheckpoint / withoutCheckpoint; - - return (withoutCheckpoint, withCheckpoint, savings * 100); - } -} - -/// -/// Context information for a checkpoint operation. -/// -/// The numeric type. -internal class CheckpointContext -{ - /// - /// The function to recompute during backward pass. - /// - public Func>? Function { get; set; } - - /// - /// The input nodes to the checkpointed function. - /// - public List> Inputs { get; set; } = new(); - - /// - /// Saved tensor values for recomputation. - /// - public Dictionary, Tensor> SavedTensors { get; set; } = new(); - - /// - /// The single output node (for single-output checkpoints). - /// - public ComputationNode? Output { get; set; } - - /// - /// The saved output value. - /// - public Tensor? OutputValue { get; set; } - - /// - /// Multiple output nodes (for multi-output checkpoints). - /// - public List>? MultiOutputs { get; set; } -} - -/// -/// Provides extension methods for gradient checkpointing on computation nodes. -/// -public static class CheckpointingExtensions -{ - /// - /// Wraps a computation with gradient checkpointing. - /// - /// The numeric type. - /// The input node. - /// The function to checkpoint. - /// The checkpointed output. - /// - /// For Beginners: A convenient way to checkpoint computations: - /// - /// - /// // Instead of: - /// var output = GradientCheckpointing<float>.Checkpoint(() => layer(input), new[] { input }); - /// - /// // You can write: - /// var output = input.WithCheckpoint(x => layer(x)); - /// - /// - /// - public static ComputationNode WithCheckpoint( - this ComputationNode input, - Func, ComputationNode> function) - { - return GradientCheckpointing.Checkpoint( - () => function(input), - new[] { input } - ); - } - - /// - /// Applies a sequence of functions with gradient checkpointing. - /// - /// The numeric type. - /// The input node. - /// The sequence of functions to apply. - /// Number of functions per checkpoint segment. - /// The final output. - public static ComputationNode WithSequentialCheckpoint( - this ComputationNode input, - IReadOnlyList, ComputationNode>> functions, - int segmentSize = 2) - { - return GradientCheckpointing.SequentialCheckpoint(functions, input, segmentSize); - } -} diff --git a/src/Diffusion/Memory/DiffusionMemoryManager.cs b/src/Diffusion/Memory/DiffusionMemoryManager.cs index 540fbf7115..09d553f39c 100644 --- a/src/Diffusion/Memory/DiffusionMemoryManager.cs +++ b/src/Diffusion/Memory/DiffusionMemoryManager.cs @@ -106,65 +106,19 @@ public DiffusionMemoryManager(DiffusionMemoryConfig? config = null, IEnumerable< #region Gradient Checkpointing Integration - /// - /// Wraps a function with gradient checkpointing for memory-efficient training. - /// - /// The function to execute with checkpointing. - /// The input computation nodes. - /// The checkpointed output node. - /// - /// - /// For Beginners: Use this to wrap expensive computations (like attention blocks). - /// - /// ```csharp - /// // Without checkpointing (stores all activations): - /// var output = attentionBlock.Forward(input); - /// - /// // With checkpointing (recomputes during backward): - /// var output = memoryManager.Checkpoint( - /// () => attentionBlock.Forward(inputNode), - /// new[] { inputNode } - /// ); - /// ``` - /// - /// - public ComputationNode Checkpoint( - Func> function, - IEnumerable> inputs) - { - if (Config.UseGradientCheckpointing) - { - return GradientCheckpointing.Checkpoint(function, inputs); - } - - // If checkpointing disabled, just execute the function - return function(); - } - - /// - /// Applies checkpointing to a sequence of layer functions. - /// - /// The sequence of layer forward functions. - /// The input node. - /// The output after all layers. - public ComputationNode CheckpointSequence( - IReadOnlyList, ComputationNode>> layers, - ComputationNode input) - { - if (Config.UseGradientCheckpointing) - { - return GradientCheckpointing.SequentialCheckpoint( - layers, input, Config.CheckpointEveryNLayers); - } - - // If checkpointing disabled, execute all layers directly - var current = input; - foreach (var layer in layers) - { - current = layer(current); - } - return current; - } + // NOTE: the ComputationNode-based Checkpoint / CheckpointSequence methods that used to sit here were + // removed along with AiDotNet.Autodiff.GradientCheckpointing. They could not save any memory: that + // helper wrapped its forward in NoGradScope, which suppresses the AiDotNet.Tensors GradientTape, but + // ComputationNode carries its own Parents/BackwardFunction graph and never consults that tape — so the + // full graph was built regardless while the recompute context was pushed and immediately discarded. The + // legacy graph has no backward driver in production at all (nothing outside Autodiff/Testing ever + // invokes BackwardFunction), and both methods had zero callers. + // + // Real activation checkpointing lives in two places, both retained: + // - AiDotNet.Tensors.Engines.Autodiff.GradientCheckpointing.Checkpoint(blockFns, input, segmentSize) + // — the tape-based primitive used by Transformer, NeuralNetworkBase, PipelineParallelModel and + // NoisePredictorBase, covered by Diffusion/CheckpointGradientEquivalenceTests. + // - ForwardWithCheckpointing below, the ILayer-based equivalent for layers outside the autodiff system. /// /// Executes a forward pass through layers with optional checkpointing. @@ -328,9 +282,14 @@ public MemoryEstimate EstimateMemory(int numLayers, long activationSizeBytes) // With checkpointing if (Config.UseGradientCheckpointing) { - var (_, withCheckpoint, _) = GradientCheckpointing.EstimateMemorySavings( - numLayers, activationSizeBytes, Config.CheckpointEveryNLayers); - estimate.WithCheckpointing = withCheckpoint; + // Segmented-checkpointing peak: one segment's activations held live during recompute, plus one + // saved activation per segment boundary. Inlined from the deleted AiDotNet.Autodiff helper (the + // arithmetic is sound and independent of that broken class): A*(s + ceil(n/s)), minimised at + // s = sqrt(n) to the familiar 2*A*sqrt(n). + int segmentSize = Math.Max(1, Config.CheckpointEveryNLayers); + int numSegments = (numLayers + segmentSize - 1) / segmentSize; + int peakSegmentLength = Math.Min(segmentSize, numLayers); + estimate.WithCheckpointing = ((long)peakSegmentLength + numSegments) * activationSizeBytes; } else { diff --git a/src/TimeSeries/AutoformerModel.cs b/src/TimeSeries/AutoformerModel.cs index 7d52154773..3399c52597 100644 --- a/src/TimeSeries/AutoformerModel.cs +++ b/src/TimeSeries/AutoformerModel.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using AiDotNet.Attributes; using AiDotNet.Autodiff; using AiDotNet.Enums; @@ -540,6 +541,42 @@ private Tensor MovingAverageEngine(Tensor x, int kernelSize, int seqLen, i return Engine.TensorMultiplyScalar(acc, _numOps.FromDouble(1.0 / kernelSize)); } + /// + /// Batched series-decomposition moving average over [B, S, D]. Identical arithmetic to the per-sample + /// , lifted one rank: narrows move from dim 0 to dim 1. + /// + /// + /// The replication padding is taken per sample from that sample's OWN first and last step (narrow on + /// dim 1), so padding never bleeds across the batch boundary — flattening to [B*S, D] and padding there + /// would splice one window's tail onto the next window's head and silently corrupt the trend. + /// + internal Tensor MovingAverageBatched(Tensor x, int kernelSize, int seqLen) + { + int leftPad = kernelSize / 2; + int rightPad = kernelSize - 1 - leftPad; + Tensor padded; + if (leftPad + rightPad == 0) + { + padded = x; + } + else + { + var front = Engine.TensorNarrow(x, 1, 0, 1); // [B, 1, D] — each sample's own first step + var back = Engine.TensorNarrow(x, 1, seqLen - 1, 1); // [B, 1, D] — each sample's own last step + var parts = new Tensor[leftPad + 1 + rightPad]; + int idx = 0; + for (int i = 0; i < leftPad; i++) parts[idx++] = front; + parts[idx++] = x; + for (int i = 0; i < rightPad; i++) parts[idx++] = back; + padded = Engine.Concat(parts, 1); + } + + Tensor acc = Engine.TensorNarrow(padded, 1, 0, seqLen); + for (int j = 1; j < kernelSize; j++) + acc = Engine.TensorAdd(acc, Engine.TensorNarrow(padded, 1, j, seqLen)); + return Engine.TensorMultiplyScalar(acc, _numOps.FromDouble(1.0 / kernelSize)); + } + // Broadcast-add a [D] bias across the sequence dim of a [S, D] tensor // (Engine.TensorAdd requires equal shapes; the FFN biases need broadcasting). private Tensor AddBias(Tensor x, Tensor bias) @@ -553,6 +590,90 @@ private Tensor AddBias(Tensor x, Tensor bias) // rolled by those delays (out[t] = Σ_i softmax(R)_i · v[(t+lag_i) mod L]). The top-k SELECTION is // data-dependent (non-differentiable, exactly as in the official implementation), but the gradient // still flows through the softmax weights (gathered R values → q, k) and through the rolled v. + // Cache for the constant diagonal-sum operator used by the matmul spectrum, keyed by (lq, lk, corrLen, d). + // Shape [corrLen, lq*lk]; ~110 KB at the default 24x24x512, and it never changes for a given model. + private readonly ConcurrentDictionary<(int Lq, int Lk, int CorrLen, int D), Tensor> _diagOperatorCache = new(); + + /// + /// Builds the constant operator A with A[lag, t*lk + (t+lag)] = 1/((corrLen-lag)*d) for t < corrLen-lag, + /// so that A @ vec(Q.K^T) is exactly the correlation spectrum R. + /// + private Tensor DiagonalMeanOperator(int lq, int lk, int corrLen, int d) + { + return _diagOperatorCache.GetOrAdd((lq, lk, corrLen, d), _ => + { + var data = new Vector(corrLen * lq * lk); + for (int lag = 0; lag < corrLen; lag++) + { + int valid = corrLen - lag; + T w = _numOps.FromDouble(1.0 / (valid * d)); + for (int t = 0; t < valid; t++) + data[lag * (lq * lk) + t * lk + (t + lag)] = w; + } + + return new Tensor(new[] { corrLen, lq * lk }, data); + }); + } + + /// + /// Correlation spectrum via a single matmul instead of a per-lag loop. + /// R[lag] = mean over (t < corrLen-lag, dim) of q[t,:]·k[t+lag,:] is, by definition, the mean of the + /// lag-th diagonal of M = Q.K^T. Summing a diagonal is LINEAR in M, so the whole spectrum is one constant + /// operator applied to vec(M): R = A @ vec(Q.K^T). + /// + /// + /// This is the op-count fix. The per-lag loop issued corrLen x ~5 dispatches (~120 at the default + /// lookback of 24) at EACH of four call sites per forward; this issues ~5 total, independent of corrLen. + /// + /// NOT bit-identical to the loop: the products are summed in a different ORDER (matmul reduction vs + /// per-lag ReduceSum), so results agree to floating-point tolerance rather than exactly. The mathematics + /// is identical — same quantity, same gradients (A is a constant, so the tape differentiates through + /// vec(M) unchanged) — only the association order differs. Equivalence is asserted at 1e-9 in + /// AutoformerBatchedEquivalenceTests; a mismatch beyond that means the formula, not the rounding, broke. + /// + internal Tensor CorrelationSpectrumMatmul(Tensor q, Tensor k, int corrLen, int d) + { + int lq = q.Shape[0]; + int lk = k.Shape[0]; + + var m = Engine.TensorMatMul(q, Engine.TensorTranspose(k)); // [lq, lk] + var flat = Engine.Reshape(m, new[] { lq * lk, 1 }); // [lq*lk, 1] + var spectrum = Engine.TensorMatMul(DiagonalMeanOperator(lq, lk, corrLen, d), flat); // [corrLen, 1] + return Engine.Reshape(spectrum, new[] { corrLen }); + } + + /// + /// Batched correlation spectrum: R[b, lag] = mean over (t < corrLen-lag, dim) of q[b,t,:]·k[b,t+lag,:], + /// for q/k shaped [B, S, D]. Returns [B, corrLen]. + /// + /// + /// This is the op-count hot spot. The per-sample path issues corrLen x ~5 ops PER SAMPLE, so a batch of + /// B costs B*corrLen*5 dispatches; four call sites per forward (encoder self x N, decoder self, decoder + /// cross) multiply that again. Computing the spectrum for the whole batch at once collapses the B factor: + /// the SAME ops run on tensors one rank wider, so each sample's arithmetic and its summation ORDER are + /// unchanged and results stay bit-identical to the per-sample path — only the dispatch count drops. + /// The top-k SELECTION is deliberately NOT batched: it is data-dependent per sample (each window picks + /// delays from its own spectrum), which is Autoformer's defining mechanism. + /// + internal Tensor CorrelationSpectrumBatched(Tensor q, Tensor k, int corrLen, int d) + { + var parts = new Tensor[corrLen]; + for (int lag = 0; lag < corrLen; lag++) + { + int valid = corrLen - lag; + // dim 1 is time for [B, S, D]; reduce over time+feature leaving the batch axis. + var qSlice = Engine.TensorNarrow(q, 1, 0, valid); + var kSlice = Engine.TensorNarrow(k, 1, lag, valid); + var summed = Engine.ReduceSum( + Engine.TensorMultiply(qSlice, kSlice), new[] { 1, 2 }, keepDims: true); // [B, 1, 1] + parts[lag] = Engine.TensorMultiplyScalar( + Engine.Reshape(summed, new[] { summed.Shape[0], 1 }), + _numOps.FromDouble(1.0 / (valid * d))); + } + + return Engine.Concat(parts, 1); // [B, corrLen] + } + private Tensor AutoCorrelationEngine(Tensor q, Tensor k, Tensor v) { int lq = q.Shape[0]; @@ -565,17 +686,11 @@ private Tensor AutoCorrelationEngine(Tensor q, Tensor k, Tensor v) topK = Math.Min(topK, corrLen); // R[lag] = mean_{t < corrLen-lag, dim} q[t]·k[t+lag], lag in [0, corrLen). Tape-tracked. - var corrParts = new Tensor[corrLen]; - for (int lag = 0; lag < corrLen; lag++) - { - int valid = corrLen - lag; - var qSlice = Engine.TensorNarrow(q, 0, 0, valid); - var kSlice = Engine.TensorNarrow(k, 0, lag, valid); - var summed = Engine.ReduceSum(Engine.TensorMultiply(qSlice, kSlice), new[] { 0, 1 }, keepDims: false); - corrParts[lag] = Engine.TensorMultiplyScalar( - Engine.Reshape(summed, new[] { 1 }), _numOps.FromDouble(1.0 / (valid * d))); - } - var corr = Engine.Concat(corrParts, 0); // [corrLen] + // Computed as ONE matmul (see CorrelationSpectrumMatmul): the spectrum is the diagonal means of + // Q.K^T, and diagonal summation is linear in that product. The previous formulation issued + // corrLen x ~5 dispatches here — ~120 at the default lookback of 24, at each of four call sites per + // forward — which is what made Autoformer training dispatch-bound. + var corr = CorrelationSpectrumMatmul(q, k, corrLen, d); // Top-k delays by correlation value (host read of the forward values; the index choice is // non-differentiable, like the paper's topk over the autocorrelation spectrum). Download the whole diff --git a/tests/AiDotNet.Tests/DirectGpuCorrectnessTests.cs b/tests/AiDotNet.Tests/DirectGpuCorrectnessTests.cs index e3591ceb09..7b033129aa 100644 --- a/tests/AiDotNet.Tests/DirectGpuCorrectnessTests.cs +++ b/tests/AiDotNet.Tests/DirectGpuCorrectnessTests.cs @@ -22,8 +22,10 @@ public DirectGpuCorrectnessTests(ITestOutputHelper output) #if !NET462 [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpu_ElementwiseOps_MatchCpu() + public async Task DirectGpu_ElementwiseOps_MatchCpu() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); if (!TryGetBackend(out var engine, out var backend)) return; using (engine) @@ -69,8 +71,10 @@ public void DirectGpu_ElementwiseOps_MatchCpu() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpu_UnaryOps_MatchCpu() + public async Task DirectGpu_UnaryOps_MatchCpu() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); if (!TryGetBackend(out var engine, out var backend)) return; using (engine) @@ -107,8 +111,10 @@ public void DirectGpu_UnaryOps_MatchCpu() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpu_Activations_MatchCpu() + public async Task DirectGpu_Activations_MatchCpu() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); if (!TryGetBackend(out var engine, out var backend)) return; using (engine) @@ -133,8 +139,10 @@ public void DirectGpu_Activations_MatchCpu() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpu_Reductions_MatchCpu() + public async Task DirectGpu_Reductions_MatchCpu() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); if (!TryGetBackend(out var engine, out var backend)) return; using (engine) @@ -160,8 +168,10 @@ public void DirectGpu_Reductions_MatchCpu() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpu_Softmax_MatchCpu() + public async Task DirectGpu_Softmax_MatchCpu() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); if (!TryGetBackend(out var engine, out var backend)) return; using (engine) @@ -222,7 +232,16 @@ private static void AssertAllClose(float[] expected, float[] actual, float tol = Assert.True(float.IsNaN(a), $"Index {i}: expected NaN, got {a}"); continue; } - Assert.InRange(a, e - tol, e + tol); + + // Mixed absolute + RELATIVE tolerance. A purely absolute bound cannot span the magnitudes these + // ops produce: Exp10 reaches ~1e5, where a single float ULP is already ~0.0078, so a fixed 1e-2 + // window failed on a GPU/CPU difference of 0.023 that is only ~2e-7 in relative terms — i.e. it + // flagged ordinary float rounding as a correctness bug. Scaling by |expected| keeps the check + // meaningful for small values while staying honest about float precision at large ones. + float allowed = tol * Math.Max(1.0f, Math.Abs(e)); + Assert.True( + Math.Abs(a - e) <= allowed, + $"Index {i}: expected {e}, got {a} (|diff| {Math.Abs(a - e)} > allowed {allowed})"); } } diff --git a/tests/AiDotNet.Tests/DirectGpuTests.cs b/tests/AiDotNet.Tests/DirectGpuTests.cs index fa773e995e..d44ffe9038 100644 --- a/tests/AiDotNet.Tests/DirectGpuTests.cs +++ b/tests/AiDotNet.Tests/DirectGpuTests.cs @@ -107,8 +107,10 @@ public async Task DirectGpuEngine_MatMul_SmallMatrix() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_MatMul_Benchmark() + public async Task DirectGpuEngine_MatMul_Benchmark() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Arrange using var engine = new DirectGpuEngine(); if (!engine.IsAvailable) @@ -155,8 +157,10 @@ public void DirectGpuEngine_MatMul_Benchmark() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_MatMul_Benchmark_KernelOnly() + public async Task DirectGpuEngine_MatMul_Benchmark_KernelOnly() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // This benchmark measures ONLY kernel compute time by: // 1. Uploading data once to GPU // 2. Running GEMM multiple times with data staying on GPU @@ -233,8 +237,10 @@ public void DirectGpuEngine_MatMul_Benchmark_KernelOnly() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_MatMul_Benchmark_KernelComparison() + public async Task DirectGpuEngine_MatMul_Benchmark_KernelComparison() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Compare simple tiled kernel vs double-buffered kernel // This helps identify if double buffering complexity is hurting performance. @@ -320,8 +326,10 @@ public void DirectGpuEngine_MatMul_Benchmark_KernelComparison() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_MatMul_Benchmark_AllKernelVariations() + public async Task DirectGpuEngine_MatMul_Benchmark_AllKernelVariations() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Comprehensive benchmark comparing ALL kernel variations to identify best configuration // This test helps identify which kernel configuration gives the best performance // by testing different hypotheses: @@ -499,8 +507,10 @@ public void DirectGpuEngine_MatMul_Benchmark_AllKernelVariations() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_Relu_Activation() + public async Task DirectGpuEngine_Relu_Activation() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Arrange using var engine = new DirectGpuEngine(); if (!engine.IsAvailable) @@ -531,8 +541,10 @@ public void DirectGpuEngine_Relu_Activation() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_Softmax() + public async Task DirectGpuEngine_Softmax() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Arrange using var engine = new DirectGpuEngine(); if (!engine.IsAvailable) @@ -603,8 +615,10 @@ public async Task HardwareCapabilities_IncludesDirectGpu() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_NewKernels_Correctness() + public async Task DirectGpuEngine_NewKernels_Correctness() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Validate that the 3 new GEMM kernels produce correct results // Test: gemm_kreg4, gemm_prefetch, gemm_wide_vec @@ -685,10 +699,14 @@ public void DirectGpuEngine_NewKernels_Correctness() { _output.WriteLine($"{name}: FAIL ({errorCount} elements with error > {tolerance}, max error: {maxError:E3})"); } + + Assert.True(errorCount == 0, + $"{name}: {errorCount} elements exceeded tolerance {tolerance} (max error: {maxError:E3})"); } catch (Exception ex) { _output.WriteLine($"{name}: ERROR - {ex.Message}"); + throw; } } @@ -698,8 +716,10 @@ public void DirectGpuEngine_NewKernels_Correctness() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_NewKernels_Benchmark() + public async Task DirectGpuEngine_NewKernels_Benchmark() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Benchmark the 3 new GEMM kernels against the current best (gemm_medium_tile) // Tests: gemm_kreg4, gemm_prefetch, gemm_wide_vec @@ -801,8 +821,10 @@ public void DirectGpuEngine_NewKernels_Benchmark() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_DynamicKernelGeneration_CompilesAndBenchmarks() + public async Task DirectGpuEngine_DynamicKernelGeneration_CompilesAndBenchmarks() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Test dynamic kernel generation like CLBlast - parameters are baked in as compile-time constants using var engine = new DirectGpuEngine(); if (!engine.IsAvailable) @@ -870,8 +892,10 @@ public void DirectGpuEngine_DynamicKernelGeneration_CompilesAndBenchmarks() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_DynamicKernelGeneration_LargeMatrices() + public async Task DirectGpuEngine_DynamicKernelGeneration_LargeMatrices() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Test dynamic kernel generation on larger matrices (2048, 4096) // This should achieve higher GFLOPS due to better occupancy using var engine = new DirectGpuEngine(); @@ -928,8 +952,10 @@ public void DirectGpuEngine_DynamicKernelGeneration_LargeMatrices() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_StaticVsDynamic_DiagnosticBenchmark() + public async Task DirectGpuEngine_StaticVsDynamic_DiagnosticBenchmark() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Diagnostic benchmark comparing static optimized kernel vs dynamic kernel system // This identifies bottlenecks and regression sources using var engine = new DirectGpuEngine(); @@ -1273,8 +1299,10 @@ public void DirectGpuEngine_ClBlast_HeadToHead_Comparison() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_ABTestKernelVariants_OptimizationComparison() + public async Task DirectGpuEngine_ABTestKernelVariants_OptimizationComparison() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Compare GEMM kernel variants: // - Variant 0: CLBlast Baseline (original implementation) // - Variant 1: XOR Swizzle (eliminates LDS bank conflicts) @@ -1328,8 +1356,10 @@ public void DirectGpuEngine_ABTestKernelVariants_OptimizationComparison() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_ComprehensiveAbTest_AllSizes() + public async Task DirectGpuEngine_ComprehensiveAbTest_AllSizes() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Runs comprehensive A/B testing across ALL sizes to identify: // 1. Which kernel variant wins at each size // 2. Bottleneck analysis (memory vs compute bound) @@ -1368,8 +1398,10 @@ public void DirectGpuEngine_ComprehensiveAbTest_AllSizes() [Fact(Timeout = 60000)] [Trait("Category", "GPU")] - public void DirectGpuEngine_GemmProfiler_RooflineAnalysis() + public async Task DirectGpuEngine_GemmProfiler_RooflineAnalysis() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); // Uses the GemmProfiler to run full profiling with roofline analysis using var engine = new DirectGpuEngine(); diff --git a/tests/AiDotNet.Tests/Fixtures/NetworkFixture.cs b/tests/AiDotNet.Tests/Fixtures/NetworkFixture.cs index 22122a6655..ce6ead0c69 100644 --- a/tests/AiDotNet.Tests/Fixtures/NetworkFixture.cs +++ b/tests/AiDotNet.Tests/Fixtures/NetworkFixture.cs @@ -18,9 +18,13 @@ namespace AiDotNet.Tests.Fixtures; /// _fixture = fixture; /// } /// +/// // Timeout is only honoured on ASYNC tests — xUnit rejects it on a synchronous method with +/// // "Tests marked with Timeout are only supported for async tests", so the test never runs at all. +/// // Keep the Timeout and make the test async; await Task.Yield() is enough. /// [Fact(Timeout = 60000)] -/// public void Test_Something() +/// public async Task Test_Something() /// { +/// await Task.Yield(); /// var network = _fixture.MiniDenseNet; /// // Use network... /// } diff --git a/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerBatchedEquivalenceTests.cs b/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerBatchedEquivalenceTests.cs new file mode 100644 index 0000000000..ddf1720595 --- /dev/null +++ b/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerBatchedEquivalenceTests.cs @@ -0,0 +1,193 @@ +using AiDotNet.LinearAlgebra; +using AiDotNet.Models.Options; +using AiDotNet.TimeSeries; +using Xunit; + +namespace AiDotNet.Tests.IntegrationTests.TimeSeries; + +/// +/// Equivalence cover for Autoformer's batched primitives. +/// +/// Autoformer trained with a PER-SAMPLE forward while its siblings (Informer, TemporalFusionTransformer) +/// use a batched one, so it issued roughly batch-times more op dispatches for identical work — measured at +/// over 10 minutes for 200 synthetic samples on the GPU engine. The batched primitives collapse that. +/// +/// The claim these tests defend is the one that actually matters for a perf change: batching must be a +/// DISPATCH optimisation only, never a numerical change. Each batched primitive runs the SAME ops one rank +/// wider, so per-sample arithmetic and summation ORDER are unchanged — results must therefore be exactly +/// equal, not merely close. The tolerance below is deliberately tight for that reason; if it ever needs +/// loosening, the batched path has changed the maths and the claim is void. +/// +public class AutoformerBatchedEquivalenceTests +{ + private const double Tolerance = 1e-12; + + private static Tensor Rand(int[] shape, int seed) + { + var rng = new Random(seed); + int total = 1; + foreach (var s in shape) total *= s; + var data = new Vector(total); + for (int i = 0; i < total; i++) data[i] = rng.NextDouble() * 2 - 1; + return new Tensor(shape, data); + } + + private static AutoformerModel Model(int embDim = 8) => + new(new AutoformerOptions + { + LookbackWindow = 12, + ForecastHorizon = 1, + EmbeddingDim = embDim, + NumEncoderLayers = 1, + NumDecoderLayers = 1, + Epochs = 1, + UseEarlyStopping = false, + }); + + /// + /// The batched correlation spectrum over [B, S, D] must equal the per-sample spectrum computed for each + /// slice independently. This is the op-count hot spot (corrLen lags x ~5 ops, x4 call sites per forward), + /// so it is where batching pays — and where a silent numerical drift would do the most damage. + /// + [Theory] + [InlineData(1, 12, 8)] + [InlineData(4, 12, 8)] + [InlineData(3, 24, 16)] + public void Batched_correlation_spectrum_equals_per_sample(int batch, int seq, int dim) + { + var model = Model(dim); + var q = Rand([batch, seq, dim], seed: 3); + var k = Rand([batch, seq, dim], seed: 7); + + var batched = model.CorrelationSpectrumBatched(q, k, seq, dim); + Assert.Equal(batch, batched.Shape[0]); + Assert.Equal(seq, batched.Shape[1]); + + // Reference: the same formula evaluated one sample at a time. + // R[b, lag] = mean over (t < seq-lag, d) of q[b,t,d] * k[b,t+lag,d] + for (int b = 0; b < batch; b++) + { + for (int lag = 0; lag < seq; lag++) + { + int valid = seq - lag; + double sum = 0.0; + for (int t = 0; t < valid; t++) + for (int d = 0; d < dim; d++) + sum += q[[b, t, d]] * k[[b, t + lag, d]]; + + double expected = sum / (valid * dim); + double actual = batched[[b, lag]]; + Assert.True( + Math.Abs(expected - actual) <= Tolerance, + $"spectrum mismatch at b={b} lag={lag}: expected {expected}, got {actual}"); + } + } + } + + /// + /// The matmul spectrum must equal the original per-lag formula. This is the shipped hot-path change: + /// R[lag] = mean over (t < corrLen-lag, dim) of q[t,:]·k[t+lag,:] is the mean of the lag-th diagonal of + /// Q·Kᵀ, and diagonal summation is linear in that product, so the whole spectrum is one constant operator + /// applied to vec(Q·Kᵀ) — ~5 dispatches instead of corrLen x ~5. + /// + /// Tolerance is 1e-9, NOT the 1e-12 used for the batched primitives, and the difference is deliberate: + /// this reformulation reassociates the summation (matmul reduction vs per-lag ReduceSum), so it is + /// mathematically identical but not bit-identical. The reference below is computed independently in plain + /// double arithmetic, so agreement means the FORMULA matches — not that two copies of the same code agree. + /// + [Theory] + [InlineData(12, 8)] + [InlineData(24, 16)] + [InlineData(24, 64)] + public void Matmul_spectrum_equals_per_lag_formula(int seq, int dim) + { + const double MatmulTolerance = 1e-9; + var model = Model(dim); + var q2 = Rand([seq, dim], seed: 5); + var k2 = Rand([seq, dim], seed: 9); + + var spectrum = model.CorrelationSpectrumMatmul(q2, k2, seq, dim); + Assert.Equal(seq, spectrum.Shape[0]); + + for (int lag = 0; lag < seq; lag++) + { + int valid = seq - lag; + double sum = 0.0; + for (int t = 0; t < valid; t++) + for (int d = 0; d < dim; d++) + sum += q2[[t, d]] * k2[[t + lag, d]]; + + double expected = sum / (valid * dim); + double actual = spectrum[[lag]]; + Assert.True( + Math.Abs(expected - actual) <= MatmulTolerance * Math.Max(1.0, Math.Abs(expected)), + $"matmul spectrum mismatch at lag={lag}: expected {expected}, got {actual}"); + } + } + + [Fact] + public void Matmul_spectrum_cache_distinguishes_correlation_length() + { + const int seq = 12; + const int dim = 8; + var model = Model(dim); + var q = Rand([seq, dim], seed: 13); + var k = Rand([seq, dim], seed: 17); + + var shortSpectrum = model.CorrelationSpectrumMatmul(q, k, corrLen: 6, d: dim); + var fullSpectrum = model.CorrelationSpectrumMatmul(q, k, corrLen: seq, d: dim); + + Assert.Equal(6, shortSpectrum.Shape[0]); + Assert.Equal(seq, fullSpectrum.Shape[0]); + } + + /// + /// The batched moving average must equal the per-sample one, and specifically must NOT let one window's + /// replication padding bleed into its neighbour. Flattening [B, S, D] to [B*S, D] and padding there would + /// splice one sample's tail onto the next sample's head — this asserts each sample is padded from its own + /// endpoints by giving neighbouring samples deliberately different levels. + /// + [Theory] + [InlineData(1, 12, 4)] + [InlineData(5, 12, 4)] + [InlineData(3, 24, 8)] + public void Batched_moving_average_equals_per_sample_and_does_not_cross_samples(int batch, int seq, int dim) + { + var model = Model(dim); + + // Give each sample a distinct offset so cross-sample contamination changes the result detectably. + var x = Rand([batch, seq, dim], seed: 11); + for (int b = 0; b < batch; b++) + for (int t = 0; t < seq; t++) + for (int d = 0; d < dim; d++) + x[[b, t, d]] = x[[b, t, d]] + (b * 100.0); + + const int kernel = 5; + var batched = model.MovingAverageBatched(x, kernel, seq); + + // Reference: replication-pad each sample from ITS OWN endpoints, then stride-1 windowed mean. + for (int b = 0; b < batch; b++) + { + int leftPad = kernel / 2, rightPad = kernel - 1 - leftPad; + for (int t = 0; t < seq; t++) + { + for (int d = 0; d < dim; d++) + { + double sum = 0.0; + for (int j = 0; j < kernel; j++) + { + int src = t + j - leftPad; + src = Math.Max(0, Math.Min(seq - 1, src)); // replication pad, per sample + sum += x[[b, src, d]]; + } + + double expected = sum / kernel; + double actual = batched[[b, t, d]]; + Assert.True( + Math.Abs(expected - actual) <= Tolerance, + $"moving-average mismatch at b={b} t={t} d={d}: expected {expected}, got {actual}"); + } + } + } + } +} diff --git a/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerTrainShapeReproTests.cs b/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerTrainShapeReproTests.cs new file mode 100644 index 0000000000..22b36326be --- /dev/null +++ b/tests/AiDotNet.Tests/IntegrationTests/TimeSeries/AutoformerTrainShapeReproTests.cs @@ -0,0 +1,290 @@ +using AiDotNet.LinearAlgebra; +using AiDotNet.Models.Options; +using AiDotNet.TimeSeries; +using Xunit; + +namespace AiDotNet.Tests.IntegrationTests.TimeSeries; + +/// +/// Regression cover for the Autoformer training crash surfaced by a research sweep: +/// +/// ArgumentException: Tensor shapes must match. Got [2048, 512] and [512, 2048]. +/// at CpuEngine.TensorAdd +/// at AutoformerModel`1.TrainCore +/// +/// Those two shapes are exactly the model's own FFN weights at the default EmbeddingDim of 512: +/// _ff1Weight is [ffDim, embeddingDim] = [2048, 512] and _ff2Weight is [embeddingDim, ffDim] = +/// [512, 2048] (ffDim = 4 * embeddingDim). TrainCore accumulates per-sample gradients with +/// accum[param] = TensorAdd(acc, g), so the throw means one FFN weight is receiving the OTHER's +/// gradient shape — the two weights' gradients are crossed. +/// +/// The forward is not obviously at fault: both use TensorMatMul(x, TensorTranspose(w)), which is +/// dimensionally correct. That points at the transpose backward failing to transpose the incoming +/// gradient back before attributing it to the parameter. +/// +/// This test uses small dims (embeddingDim 8 => ffDim 32) so the same crossing appears as +/// [32, 8] vs [8, 32] and runs in seconds instead of minutes. It asserts only that training +/// COMPLETES — a shape-crossed gradient throws, so completion is the signal. +/// +[Collection("EngineCurrentGlobalState")] +public class AutoformerTrainShapeReproTests +{ + private static (Matrix X, Vector Y) SyntheticSeries(int n, int features, int seed = 11) + { + var rng = new Random(seed); + var x = new Matrix(n, features); + var y = new Vector(n); + double level = 0.0; + for (int i = 0; i < n; i++) + { + // Mean-reverting series so the model has real (not degenerate) signal to fit. + level = 0.6 * level + (rng.NextDouble() - 0.5) * 0.02; + y[i] = level; + for (int f = 0; f < features; f++) + { + x[i, f] = i > f ? y[i - f - 1] : 0.0; + } + } + + return (x, y); + } + + // embeddingDim 8 (ffDim 32) passes, so this is NOT a universal FFN crossing — it is dimension + // dependent. BackwardFunctions carries size-thresholded specialised paths, so the real default of + // 512 (ffDim 2048) is tested explicitly rather than assumed to behave like the small case. + [Theory] + [InlineData(8)] + [InlineData(64)] + [InlineData(512)] // the production default — the configuration that crashed the sweep + public void Autoformer_trains_without_crossing_its_ffn_gradient_shapes(int embeddingDim) + { + var (x, y) = SyntheticSeries(n: 160, features: 3); + + var options = new AutoformerOptions + { + LookbackWindow = 12, + ForecastHorizon = 1, + // ffDim is 4 * EmbeddingDim. A crossed gradient shows up as "Got [4d, d] and [d, 4d]" — + // e.g. [32, 8] at d=8, or [2048, 512] at the default d=512 seen in the sweep. + EmbeddingDim = embeddingDim, + NumEncoderLayers = 1, + NumDecoderLayers = 1, + Epochs = 2, + UseEarlyStopping = false, + }; + + var model = new AutoformerModel(options); + + // A crossed FFN gradient throws ArgumentException out of TrainCore's accumulation step. + var ex = Record.Exception(() => model.Train(x, y)); + + Assert.True( + ex is null, + $"Autoformer training threw {ex?.GetType().Name}: {ex?.Message}. " + + "A 'Tensor shapes must match' here means the FFN weights' gradients are crossed " + + "(ff1 is [ffDim, embDim], ff2 is [embDim, ffDim])."); + } + + /// + /// The sweep crashed on AutoformerModel<FLOAT>, and float is precisely the type that unlocks the + /// resident/fused specialised paths in the engine (roughly 143 typeof(T)==float gates) which a double + /// model never touches. The double cases above all pass at the same dimensions, so this reproduces the + /// sweep's exact configuration: float, LookbackWindow 24, default EmbeddingDim/layer counts. + /// + [Fact] + public void Autoformer_float_trains_at_the_sweep_configuration() + { + var (xd, yd) = SyntheticSeries(n: 200, features: 3); + var x = new Matrix(xd.Rows, xd.Columns); + for (int i = 0; i < xd.Rows; i++) + for (int j = 0; j < xd.Columns; j++) + x[i, j] = (float)xd[i, j]; + var y = new Vector(yd.Length); + for (int i = 0; i < yd.Length; i++) y[i] = (float)yd[i]; + + // Mirrors Ooples' AutoformerFloatForecaster exactly: only these options are set, so + // EmbeddingDim (512) and the encoder/decoder layer counts stay at their defaults. + var options = new AutoformerOptions + { + LookbackWindow = 24, + ForecastHorizon = 1, + Epochs = 2, + UseEarlyStopping = true, + EarlyStoppingPatience = 5, + }; + + var model = new AutoformerModel(options); + var ex = Record.Exception(() => model.Train(x, y)); + + Assert.True( + ex is null, + $"Autoformer training threw {ex?.GetType().Name}: {ex?.Message}. " + + "Expected shapes of the form [2048, 512] vs [512, 2048] if the FFN gradients are crossed."); + } + + /// + /// The sweep trained THREE cells CONCURRENTLY (OOPLES_RESEARCH_PARALLELISM=3) against one + /// process-global engine. Two different Autoformer instances each own an _ff1Weight [ffDim, embDim] + /// and an _ff2Weight [embDim, ffDim], so any process-global state keyed such that instances collide + /// would add one instance's ff1 gradient to another's ff2 — producing exactly + /// "Got [2048, 512] and [512, 2048]", and only intermittently (timing dependent), which matches the + /// sweep crashing one cell while others merely ran slow. + /// + /// Every single-threaded configuration passes (dims 8/64/512, double and float, CPU engine and GPU + /// engine), so concurrency is the remaining untested differentiator. Small dims are used deliberately: + /// a cross-instance collision does not depend on size, so this stays fast. + /// + [Fact] + public void Autoformer_concurrent_instances_do_not_cross_gradients() + { + const int instances = 3; + var errors = new System.Collections.Concurrent.ConcurrentBag(); + + Parallel.For(0, instances, i => + { + try + { + var (x, y) = SyntheticSeries(n: 160, features: 3, seed: 11 + i); + var options = new AutoformerOptions + { + LookbackWindow = 12, + ForecastHorizon = 1, + EmbeddingDim = 8, + NumEncoderLayers = 1, + NumDecoderLayers = 1, + Epochs = 2, + UseEarlyStopping = false, + }; + new AutoformerModel(options).Train(x, y); + } + catch (Exception ex) + { + errors.Add(ex); + } + }); + + Assert.True( + errors.IsEmpty, + "Concurrent Autoformer training threw: " + + string.Join(" | ", errors.Select(e => e.GetType().Name + ": " + e.Message))); + } + + /// + /// The sweep's ACTUAL condition: concurrency AND the GPU engine AND float, together. Every simpler + /// combination now passes — dims 8/64/512, double and float, CPU engine, GPU engine single-threaded, + /// and concurrency on the CPU engine. This is the last combination short of real FCHL data. + /// Dims are reduced to 64 because a cross-instance state collision does not depend on size, and 512 + /// costs ~8 minutes per instance. + /// + [SkippableFact] + public void Autoformer_concurrent_float_on_gpu_does_not_cross_gradients() + { + var restore = AiDotNet.Tensors.Engines.AiDotNetEngine.Current; + try + { + var gpuEngine = new AiDotNet.Tensors.Engines.DirectGpuTensorEngine(); + Skip.IfNot(gpuEngine.SupportsGpu, "no CUDA GPU available on this host"); + AiDotNet.Tensors.Engines.AiDotNetEngine.Current = gpuEngine; + + var errors = new System.Collections.Concurrent.ConcurrentBag(); + Parallel.For(0, 3, i => + { + try + { + var (xd, yd) = SyntheticSeries(n: 160, features: 3, seed: 11 + i); + var x = new Matrix(xd.Rows, xd.Columns); + for (int r = 0; r < xd.Rows; r++) + for (int c = 0; c < xd.Columns; c++) + x[r, c] = (float)xd[r, c]; + var y = new Vector(yd.Length); + for (int r = 0; r < yd.Length; r++) y[r] = (float)yd[r]; + + var options = new AutoformerOptions + { + LookbackWindow = 24, + ForecastHorizon = 1, + EmbeddingDim = 64, + Epochs = 2, + UseEarlyStopping = false, + }; + new AutoformerModel(options).Train(x, y); + } + catch (Exception ex) + { + errors.Add(ex); + } + }); + + Assert.True( + errors.IsEmpty, + "Concurrent float Autoformer on GPU threw: " + + string.Join(" | ", errors.Select(e => e.GetType().Name + ": " + e.Message))); + } + finally + { + AiDotNet.Tensors.Engines.AiDotNetEngine.Current = restore; + } + } + + /// + /// The sweep ran with the GPU engine ADOPTED, and that is the last untested differentiator: the CPU + /// cases above all pass. DirectGpuTensorEngine derives from CpuEngine, so the CpuEngine.TensorAdd frame + /// in the crash stack is exactly what a GPU-engine fallback looks like, and float unlocks the resident / + /// fused specialisations that a CPU-engine run never enters. + /// Skips (rather than fails) when no GPU is present so CI stays green on GPU-less runners. + /// + [SkippableFact] + public void Autoformer_float_trains_with_the_gpu_engine_adopted() + { + var restore = AiDotNet.Tensors.Engines.AiDotNetEngine.Current; + var restoreLogger = AiDotNet.Tensors.Engines.AiDotNetEngine.Logger; + bool gpu = false; + var log = new System.Text.StringBuilder(); + try + { + // Capture the engine's OWN reason rather than inferring from the bool: it distinguishes + // "no device" from "detected but failed its correctness probe" from "init threw", and those + // three point at completely different fixes. + AiDotNet.Tensors.Engines.AiDotNetEngine.Logger = m => log.AppendLine(m); + + // Construct the GPU engine DIRECTLY rather than going through AutoDetectAndConfigureGpu. + // AutoDetect additionally runs a correctness probe and allocator registration, either of which + // can veto adoption — so it cannot distinguish "no device" from "device rejected". Building the + // engine and reading SupportsGpu isolates device availability, which is what this test needs. + var gpuEngine = new AiDotNet.Tensors.Engines.DirectGpuTensorEngine(); + gpu = gpuEngine.SupportsGpu; + Skip.IfNot(gpu, $"DirectGpuTensorEngine reports no GPU. Engine log: {log.ToString().Trim()}"); + AiDotNet.Tensors.Engines.AiDotNetEngine.Current = gpuEngine; + + var (xd, yd) = SyntheticSeries(n: 200, features: 3); + var x = new Matrix(xd.Rows, xd.Columns); + for (int i = 0; i < xd.Rows; i++) + for (int j = 0; j < xd.Columns; j++) + x[i, j] = (float)xd[i, j]; + var y = new Vector(yd.Length); + for (int i = 0; i < yd.Length; i++) y[i] = (float)yd[i]; + + var options = new AutoformerOptions + { + LookbackWindow = 24, + ForecastHorizon = 1, + Epochs = 2, + UseEarlyStopping = true, + EarlyStoppingPatience = 5, + }; + + var model = new AutoformerModel(options); + var ex = Record.Exception(() => model.Train(x, y)); + + Assert.True( + ex is null, + $"Autoformer on the GPU engine threw {ex?.GetType().Name}: {ex?.Message}. " + + "Shapes like [2048, 512] vs [512, 2048] mean the FFN weights' gradients are crossed."); + } + finally + { + AiDotNet.Tensors.Engines.AiDotNetEngine.Logger = restoreLogger; + AiDotNet.Tensors.Engines.AiDotNetEngine.Current = restore; + } + } +} diff --git a/tests/AiDotNet.Tests/Performance/SenseVoiceTrainStepProfile.cs b/tests/AiDotNet.Tests/Performance/SenseVoiceTrainStepProfile.cs index af21f97b9b..3f2423f515 100644 --- a/tests/AiDotNet.Tests/Performance/SenseVoiceTrainStepProfile.cs +++ b/tests/AiDotNet.Tests/Performance/SenseVoiceTrainStepProfile.cs @@ -12,6 +12,7 @@ // that exercises SenseVoice.Train. using System.Collections.Generic; +using System.Threading.Tasks; using System.Diagnostics; using AiDotNet.LinearAlgebra; using AiDotNet.NeuralNetworks; @@ -42,8 +43,10 @@ public class SenseVoiceTrainStepProfile private const double ForwardForTrainingBudgetMs = 2_000; [Fact(Timeout = 600000)] - public void Profile_StepBreakdown() + public async Task Profile_StepBreakdown() { + // Timeout requires an async test; yielding makes this legal without changing the work below. + await Task.Yield(); AiDotNetEngine.ResetToCpu(); // SenseVoice-Small paper-faithful defaults (Du et al. 2024) diff --git a/tests/AiDotNet.Tests/UnitTests/Diffusion/MemoryManagementTests.cs b/tests/AiDotNet.Tests/UnitTests/Diffusion/MemoryManagementTests.cs index bf84a5603d..9173088021 100644 --- a/tests/AiDotNet.Tests/UnitTests/Diffusion/MemoryManagementTests.cs +++ b/tests/AiDotNet.Tests/UnitTests/Diffusion/MemoryManagementTests.cs @@ -13,6 +13,43 @@ namespace AiDotNet.Tests.UnitTests.Diffusion; /// public class MemoryManagementTests : DiffusionUnitTestBase { + #region DiffusionMemoryManager Tests + + [Theory] + [InlineData(0)] + [InlineData(-4)] + public void EstimateMemory_NormalizesNonPositiveCheckpointIntervals(int interval) + { + var manager = new DiffusionMemoryManager(new DiffusionMemoryConfig + { + UseGradientCheckpointing = true, + UseActivationPooling = false, + CheckpointEveryNLayers = interval, + }); + + MemoryEstimate estimate = manager.EstimateMemory(numLayers: 8, activationSizeBytes: 100); + + Assert.Equal(800, estimate.WithoutCheckpointing); + Assert.Equal(900, estimate.WithCheckpointing); + } + + [Fact] + public void EstimateMemory_ClampsPeakSegmentToLayerCount() + { + var manager = new DiffusionMemoryManager(new DiffusionMemoryConfig + { + UseGradientCheckpointing = true, + UseActivationPooling = false, + CheckpointEveryNLayers = 100, + }); + + MemoryEstimate estimate = manager.EstimateMemory(numLayers: 8, activationSizeBytes: 100); + + Assert.Equal(900, estimate.WithCheckpointing); + } + + #endregion + #region ActivationPool Tests [Fact(Timeout = 120000)]