From 74af37d67014f7111003471a7677709e11b8e8df Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 07:56:46 -0400 Subject: [PATCH 01/29] fix(stats): break BasicStats.CalculateStats recursion that crashed test host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BasicStats's lazy-stats accessors all read through property getters that call EnsureFullStatsComputed -> CalculateStats. When CalculateStats itself reads any of those properties (N, Mean, Variance, StandardDeviation, Median, FirstQuartile, ThirdQuartile), the getter re-enters EnsureFullStatsComputed because _fullStatsComputed is still false during the body of CalculateStats — that flag is only set after CalculateStats returns. The result is unbounded recursion that crashes the xUnit test host with a StackOverflowException. Stack from CI failures: BasicStats.CalculateStats(Vector) BasicStats.EnsureFullStatsComputed() BasicStats.get_N() // <-- re-entry BasicStats.CalculateStats(Vector) ... Reported as the "Test Run Aborted — host process exited unexpectedly" on these CI jobs (PR #1154 / master): - AiDotNet.Serving.Tests - ModelFamily - Classification - ModelFamily - Clustering/GP - ModelFamily - Regression - ModelFamily - TimeSeries/Activation/Loss - Unit - 04 Feature/Fit/Fitness/Genetics Fix: compute every intermediate value into a local variable, only assign to the publicly-observable properties at the end. Property reads never happen inside CalculateStats, so the lazy getter never re-enters. Verified locally: FederatedRun_Lifecycle_FedAvg_AggregatesAndAdvancesRound (which serializes a model and triggers the lazy stats path) now passes end-to-end instead of crashing the host. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- src/Statistics/BasicStats.cs | 46 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/src/Statistics/BasicStats.cs b/src/Statistics/BasicStats.cs index 2d52203bd4..1fbc2763cc 100644 --- a/src/Statistics/BasicStats.cs +++ b/src/Statistics/BasicStats.cs @@ -436,18 +436,40 @@ public static BasicStats Empty() /// private void CalculateStats(Vector values) { - N = values.Length; - if (N == 0) return; - Mean = values.Average(); - Variance = values.Variance(); - StandardDeviation = _numOps.Sqrt(Variance); - (Skewness, Kurtosis) = StatisticsHelper.CalculateSkewnessAndKurtosis(values, Mean, StandardDeviation, N); - Min = values.Min(); - Max = values.Max(); - Median = StatisticsHelper.CalculateMedian(values); - (FirstQuartile, ThirdQuartile) = StatisticsHelper.CalculateQuantiles(values); - InterquartileRange = _numOps.Subtract(ThirdQuartile, FirstQuartile); - MAD = StatisticsHelper.CalculateMeanAbsoluteDeviation(values, Median); + // CRITICAL: every property on this class (N, Mean, Variance, + // StandardDeviation, Median, etc.) goes through a getter that calls + // EnsureFullStatsComputed(), which calls back into this method + // when _fullStatsComputed is false. We are exactly in that window + // here, so reading any property would recurse forever and crash + // the test host with a StackOverflowException. Compute everything + // into locals and assign to the properties at the end. + int n = values.Length; + N = n; + if (n == 0) return; + + T mean = values.Average(); + T variance = values.Variance(); + T stdDev = _numOps.Sqrt(variance); + var (skewness, kurtosis) = StatisticsHelper.CalculateSkewnessAndKurtosis(values, mean, stdDev, n); + T min = values.Min(); + T max = values.Max(); + T median = StatisticsHelper.CalculateMedian(values); + var (firstQuartile, thirdQuartile) = StatisticsHelper.CalculateQuantiles(values); + T iqr = _numOps.Subtract(thirdQuartile, firstQuartile); + T mad = StatisticsHelper.CalculateMeanAbsoluteDeviation(values, median); + + Mean = mean; + Variance = variance; + StandardDeviation = stdDev; + Skewness = skewness; + Kurtosis = kurtosis; + Min = min; + Max = max; + Median = median; + FirstQuartile = firstQuartile; + ThirdQuartile = thirdQuartile; + InterquartileRange = iqr; + MAD = mad; } /// From f43da3379898c57da968b8ac5c9dde20c31fb4c7 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 08:02:22 -0400 Subject: [PATCH 02/29] test(data): cross-platform retry trigger for RobustFileOps tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two RobustFileOps retry tests passed on Windows but failed on the Linux CI runner because FileShare.None on a FileStream does not actually block File.Move on POSIX: - Move_SucceedsAfter_TransientSharingViolation - Move_Propagates_WhenLockNeverReleases Both used a held FileStream with FileShare.None as the "failed-attempt" trigger. On Linux that does not block rename(2), so File.Move succeeded on the first attempt — Move_Propagates' Assert. Throws fired ("No exception was thrown") and Move_SucceedsAfter short-circuited without ever exercising the retry loop. Replaced the lock-based simulation with a cross-platform missing- parent-directory trigger: - Move_SucceedsAfter_TransientSharingViolation: destination's parent directory does not exist when MoveWithRetryAsync runs. File.Move throws DirectoryNotFoundException (an IOException subclass) on each attempt. A background task creates the parent ~250 ms in, so a subsequent attempt succeeds. Retry path is exercised on every platform. - Move_Propagates_WhenLockNeverReleases: parent directory is never created. Every attempt throws DirectoryNotFoundException; the final attempt must propagate. Test now asserts the more specific DirectoryNotFoundException type for clarity, and adds a check that the source file is still in place after the failed move (the move never started, so src must remain). Verified locally: all 5 RobustFileOpsMoveRetryTests pass on net10.0. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../Data/RobustFileOpsMoveRetryTests.cs | 81 +++++++++---------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs index c120d52a7f..7345168f1e 100644 --- a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs +++ b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs @@ -46,56 +46,43 @@ public async Task Move_Succeeds_WhenNoContention() } /// - /// Simulates the Windows antivirus / indexer lock pattern: open the source - /// file with exclusive sharing for a short window, then release it. - /// The retry loop must tolerate the transient IOException and succeed - /// once the lock is released. + /// Cross-platform retry-trigger: the destination's parent directory + /// doesn't exist when the move starts, so File.Move throws + /// DirectoryNotFoundException (a subclass of IOException). A + /// background task creates the parent ~250 ms later, after which a + /// retry attempt succeeds. This exercises the same retry path as + /// the Windows AV / indexer scenario without depending on Windows- + /// specific FileShare.None semantics — on Linux, opening a + /// FileStream with FileShare.None does not actually block File.Move, + /// so the original lock-based simulation passed trivially without + /// ever exercising retry on the Linux CI runner. /// [Fact] public async Task Move_SucceedsAfter_TransientSharingViolation() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); - string dst = Path.Combine(Path.GetTempPath(), $"aidotnet_move_dst_{Guid.NewGuid()}.bin"); + string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}"); + string dst = Path.Combine(parentDir, "dst.bin"); File.WriteAllBytes(src, [9, 9, 9]); - // Acquire an exclusive handle on the source, synchronize with the - // main test so the retry path is actually exercised (otherwise - // Task.Run scheduling could let the main thread call File.Move - // before the lock exists and the test passes trivially without - // ever triggering retry), hold the handle for ~250 ms, then - // release so the retry loop eventually succeeds. Retry delay is - // scaled up to 100 ms per attempt here to keep the race window - // deterministic; production default is 200 ms. - var lockAcquired = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var lockRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + // Background task creates the parent directory after ~250 ms, + // letting the retry loop succeed once the directory exists. + // Retry schedule: 100, 200, 300, 400 ms (initialDelayMs * attempt), + // so by attempt 3 or 4 the directory will be in place. + var dirReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _ = Task.Run(async () => { try { - using var holder = new FileStream(src, FileMode.Open, FileAccess.Read, FileShare.None); - lockAcquired.TrySetResult(true); await Task.Delay(TimeSpan.FromMilliseconds(250)); - } - catch (Exception ex) - { - // If the FileStream open itself fails, release both - // synchronization primitives so the main test fails fast - // with a clear error rather than hanging on the wait. - lockAcquired.TrySetException(ex); + Directory.CreateDirectory(parentDir); } finally { - lockRelease.TrySetResult(true); + dirReady.TrySetResult(true); } }); - // Wait for the background task to actually acquire the exclusive - // handle before we start the move. Without this await the test - // is a race between Task.Run scheduling and the main thread, and - // the retry path would not be exercised on the "fast" side of - // the race — a silently-passing test. - await lockAcquired.Task; - try { await RobustFileOps.MoveWithRetryAsync(src, dst, CancellationToken.None, maxAttempts: 8, initialDelayMs: 100); @@ -105,14 +92,12 @@ public async Task Move_SucceedsAfter_TransientSharingViolation() } finally { - // Drain the background task before teardown regardless of the - // assertion outcome. If we didn't await here and an assertion - // above threw, the background task could still own the - // FileStream when File.Delete(src) runs, masking the real - // failure with a confusing "file in use" IOException. - await lockRelease.Task; + // Drain the background task before teardown so cleanup is + // deterministic regardless of which assertion threw. + await dirReady.Task; if (File.Exists(src)) File.Delete(src); if (File.Exists(dst)) File.Delete(dst); + if (Directory.Exists(parentDir)) Directory.Delete(parentDir, recursive: true); } } @@ -175,29 +160,35 @@ public void ReplaceSync_Succeeds_WhenNoContention() /// /// If every retry attempt fails, the final attempt must propagate the /// underlying exception rather than silently succeed or hang. + /// + /// Cross-platform retry-trigger: destination's parent directory never + /// exists, so every File.Move attempt throws DirectoryNotFoundException + /// (an IOException subclass). Assert.ThrowsAsync<IOException> + /// matches that subtype. /// [Fact] public async Task Move_Propagates_WhenLockNeverReleases() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); - string dst = Path.Combine(Path.GetTempPath(), $"aidotnet_move_dst_{Guid.NewGuid()}.bin"); + string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}"); + string dst = Path.Combine(parentDir, "dst.bin"); File.WriteAllBytes(src, [1]); - // Hold an exclusive read handle for the duration of the call — all - // attempts will fail with IOException and the final one should - // surface. - using var holder = new FileStream(src, FileMode.Open, FileAccess.Read, FileShare.None); + // Parent directory is never created — every retry attempt fails, + // and the final attempt must propagate the IOException to the + // caller (rather than swallow it). try { - await Assert.ThrowsAsync( + await Assert.ThrowsAsync( () => RobustFileOps.MoveWithRetryAsync(src, dst, CancellationToken.None, maxAttempts: 3, initialDelayMs: 1)); Assert.False(File.Exists(dst), "Destination must not exist when the move fails"); + Assert.True(File.Exists(src), "Source must remain in place when the move fails"); } finally { - holder.Dispose(); if (File.Exists(src)) File.Delete(src); if (File.Exists(dst)) File.Delete(dst); + if (Directory.Exists(parentDir)) Directory.Delete(parentDir, recursive: true); } } } From abcdcdd13eec4e84789e834da3d33f10ecc14542 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 08:45:13 -0400 Subject: [PATCH 03/29] fix(serialization): match MultiHeadAttentionLayer 5-arg constructor in deserializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeserializationHelper.CreateMultiHeadAttentionLayer was looking up a 4-parameter constructor signature (int, int, int, IActivationFunction) but MultiHeadAttentionLayer's constructor is actually 5-parameter: (int, int, int, IActivationFunction?, IInitializationStrategy?) Type.GetConstructor matches by exact parameter list, not by "first N plus defaults," so the lookup returned null and threw "Cannot find MultiHeadAttentionLayer constructor with (int, int, int, IActivationFunction)" Failure path observed in CI: - InferenceOptimizer.OptimizeForInference(model, cloneModel: true) -> NeuralNetworkBase.Clone (serialization round-trip) -> DeserializationHelper.CreateMultiHeadAttentionLayer (throws) -> caught in OptimizeForInference, returns (model, false) - Test InferenceOptimizer_RewritesMultiHeadAttention_To CachedAttention_ForTextGeneration_WhenKVCacheEnabled then sees anyApplied == false instead of the expected rewrite. The fix mirrors how CreateDenseLayer already passes IInitializationStrategy in its constructor lookup. Pass null for the strategy slot, matching the constructor's default-value semantics. Verified locally: all 9 InferenceOptimizerTests pass on net10.0. Wider impact: this also unblocks Clone-via-serialization for any model containing MHA layers — previously every transformer-style model would silently skip inference optimizations after clone failed. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- src/Helpers/DeserializationHelper.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Helpers/DeserializationHelper.cs b/src/Helpers/DeserializationHelper.cs index 2f16126167..4991df29d0 100644 --- a/src/Helpers/DeserializationHelper.cs +++ b/src/Helpers/DeserializationHelper.cs @@ -989,7 +989,16 @@ private static object CreateDenseLayer(Type type, int[] inputShape, int[] out private static object CreateMultiHeadAttentionLayer(Type type, int[] inputShape, Dictionary? additionalParams) { - // MultiHeadAttentionLayer(int sequenceLength, int embeddingDimension, int headCount, IActivationFunction? activationFunction = null) + // MultiHeadAttentionLayer(int sequenceLength, int embeddingDimension, + // int headCount, IActivationFunction? activationFunction = null, + // IInitializationStrategy? initializationStrategy = null) + // + // The optional initializationStrategy parameter MUST be included in + // the reflection signature even though it has a default — Type.GetConstructor + // matches by exact parameter list, not by "first N + defaults". Without it + // the lookup returns null and Clone-via-serialization (used by + // InferenceOptimizer.OptimizeForInference and any model save/load + // round-trip) fails on every transformer with an MHA layer. if (inputShape.Length < 2) { throw new InvalidOperationException("MultiHeadAttentionLayer requires input shape [sequenceLength, embeddingDimension]."); @@ -1000,14 +1009,15 @@ private static object CreateMultiHeadAttentionLayer(Type type, int[] inputSha int headCount = TryGetInt(additionalParams, "HeadCount") ?? ResolveDefaultHeadCount(embDim); var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); - var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType }); + var initStrategyType = typeof(IInitializationStrategy<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType, initStrategyType }); if (ctor is null) { - throw new InvalidOperationException("Cannot find MultiHeadAttentionLayer constructor with (int, int, int, IActivationFunction)."); + throw new InvalidOperationException("Cannot find MultiHeadAttentionLayer constructor with (int, int, int, IActivationFunction, IInitializationStrategy)."); } object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); - return ctor.Invoke(new object?[] { seqLen, embDim, headCount, activation }); + return ctor.Invoke(new object?[] { seqLen, embDim, headCount, activation, null }); } private static object CreateFlashAttentionLayer(Type type, int[] inputShape, Dictionary? additionalParams) From d61cc69fffc8d5e58a349b21cdeee53c0809d106 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 09:45:34 -0400 Subject: [PATCH 04/29] fix(optimizer): re-allocate Adam moments when cached shape mismatches param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AdamOptimizer.Step keyed its per-parameter moment tensors (_tapeM, _tapeV) by Tensor reference. If a parameter was first seen while a lazy-initialized layer (e.g. MultiHeadAttentionLayer with IsLazy: true initialization strategy) had its weights allocated as the placeholder [0, 0] tensor, the cached m / v captured shape [0, 0] and Length 0. Once the layer materialized real weights and real-shape gradients arrived, mScaled and gradScaled differed in shape; TensorAdd broadcast to the larger shape and the result no longer matched m's underlying buffer. Fix: at every Step, validate the cached m and v match the parameter's current shape via SequenceEqual, and re-allocate if not. Identity caching by reference still works for stable parameters; the explicit shape check covers the lazy-init case. Note: this fix alone is not sufficient to make MobileNetV3_Train_CompletesWithoutError pass — that test also hits a separate bug in AiDotNet.Tensors (CpuEngine.TensorCopy uses sourceArray.Length instead of source.Length, see follow-up PR on the Tensors repo). This commit fixes the lazy-init half of the issue, which would otherwise mask the Tensors bug behind a noisier symptom. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- src/Optimizers/AdamOptimizer.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Optimizers/AdamOptimizer.cs b/src/Optimizers/AdamOptimizer.cs index cbc0c97248..96687a522d 100644 --- a/src/Optimizers/AdamOptimizer.cs +++ b/src/Optimizers/AdamOptimizer.cs @@ -482,13 +482,21 @@ public override void Step(TapeStepContext context) if (!context.Gradients.TryGetValue(param, out var grad)) continue; - // Lazily initialize per-parameter moment tensors - if (!_tapeM.TryGetValue(param, out var m)) + // Lazily initialize per-parameter moment tensors. If the parameter + // was first seen while a lazy-init layer (e.g. + // MultiHeadAttentionLayer with IsLazy: true initialization + // strategy) still had its weights allocated as a placeholder + // [0, 0] tensor, our cached m / v captured the placeholder shape. + // Once the layer materializes real weights, the gradient arrives + // at the real shape — m / v need to be re-allocated to match, + // otherwise TensorAdd's result has a length larger than m and + // TensorCopy throws "Destination array was not long enough". + if (!_tapeM.TryGetValue(param, out var m) || !m._shape.SequenceEqual(param._shape)) { m = new Tensor(param._shape); _tapeM[param] = m; } - if (!_tapeV.TryGetValue(param, out var v)) + if (!_tapeV.TryGetValue(param, out var v) || !v._shape.SequenceEqual(param._shape)) { v = new Tensor(param._shape); _tapeV[param] = v; From a38e3a00a7b38a18256030e6fcbe068d98d7bdef Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 09:54:44 -0400 Subject: [PATCH 05/29] fix(serving): cross-platform sanitizer for AesGcm artifact filenames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Path.GetInvalidFileNameChars returns a platform-specific set: - Windows: includes ':', '\', '*', '?', '<', '>', '|', '"' plus control chars 1-31 - Linux / macOS: only '\0' and '/' Encrypted model artifacts are designed to be portable across operating systems (an artifact written on a Linux training cluster might be loaded on a Windows inference host). Using the platform-specific set broke the AesGcmModelArtifactProtectorTests. ProtectToFile_WritesHeaderAndReturnsArtifact test on Linux CI: expected "my_model.aidn.enc" actual "my:model.aidn.enc" (':' isn't invalid on POSIX) Fix: replace Path.GetInvalidFileNameChars with a hardcoded cross-platform-invalid set that combines the Windows superset with POSIX. Now the sanitizer produces identical output on every OS, so artifacts are guaranteed mountable everywhere. Verified locally: ProtectToFile_WritesHeaderAndReturnsArtifact passes on net10.0. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../Services/AesGcmModelArtifactProtector.cs | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs index 77be8bffe9..29a6b1ca38 100644 --- a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs +++ b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs @@ -72,10 +72,27 @@ public ProtectedModelArtifact ProtectToFile(string modelName, string sourcePath, } } + /// + /// Cross-platform-invalid filename characters. Combines the Windows + /// invalid set (most restrictive: ":" + "\\" + reserved punctuation + + /// control chars) with POSIX "/" and "\0". Used instead of + /// because that method + /// returns a platform-specific set — on Linux it only contains '\0' + /// and '/', so a model name like "my:model" sanitizes to "my:model" + /// on Linux but "my_model" on Windows. Encrypted artifacts are + /// designed to be portable, so we apply the strict Windows superset + /// on every OS to guarantee the output is mountable everywhere. + /// + private static readonly HashSet CrossPlatformInvalidFileNameChars = + new(new[] + { + '\0', '/', '\\', ':', '*', '?', '"', '<', '>', '|', + } + .Concat(Enumerable.Range(1, 31).Select(i => (char)i))); + private static string SanitizeFileName(string name) { - var invalid = Path.GetInvalidFileNameChars(); - var chars = name.Select(c => invalid.Contains(c) ? '_' : c).ToArray(); + var chars = name.Select(c => CrossPlatformInvalidFileNameChars.Contains(c) ? '_' : c).ToArray(); return new string(chars); } } From d9eeebeb32892e81d2e32b1a81f35552b105f522 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 15:56:57 -0400 Subject: [PATCH 06/29] fix(layers): sparselinearlayer reports supportstraining true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The layer's SupportsTraining property previously returned false with a detailed comment explaining that sparse weight tensors don't fit the tape's dense ParameterBuffer contract. But returning false was incorrect: SupportsTraining gates the LEGACY non-tape training path (`if (layer.SupportsTraining) layer.UpdateParameters(lr)`), and the layer DOES have a working UpdateParameters that updates both the sparse weight tensor and the dense bias vector from gradients computed in Backward. Setting it to false was preventing the layer from training in the legacy path even though the update mechanism existed. Tape-mode discovery is unaffected by SupportsTraining — that path uses [TrainableParameter] / RegisterTrainableParameter discovery, not this property. The sparse weight tensor remains invisible to tape mode pending sparse-aware ParameterBuffer support, which is a separate architectural follow-up. Updated docstring to describe the actual semantics (legacy path trains the layer; tape-mode caveat documented inline). Verified locally: SparseLinearLayer_SupportsTraining_IsTrue passes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Layers/SparseLinearLayer.cs | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/NeuralNetworks/Layers/SparseLinearLayer.cs b/src/NeuralNetworks/Layers/SparseLinearLayer.cs index 88f7d95f1f..d3076952ff 100644 --- a/src/NeuralNetworks/Layers/SparseLinearLayer.cs +++ b/src/NeuralNetworks/Layers/SparseLinearLayer.cs @@ -101,21 +101,26 @@ public partial class SparseLinearLayer : LayerBase _weights.NonZeroCount + OutputFeatures; /// - /// Gets whether this layer supports training. Returns false because this layer's - /// sparse weight tensor is not exposed to the tape-based autodiff path: the layer has no - /// [TrainableParameter] fields and never calls RegisterTrainableParameter, - /// so TapeTrainingStep.CollectParameters finds zero parameters here. In tape mode - /// (the default), the layer would be silently skipped while its peers train — producing - /// a partially-trainable network that is almost never the intended behavior. + /// Gets whether this layer supports training. Returns true: the layer + /// owns a working that updates its sparse + /// weight tensor and dense bias vector from gradients computed in + /// , and the legacy training path + /// (if (layer.SupportsTraining) layer.UpdateParameters(lr)) trains + /// the layer correctly. Biases are also registered as a tape trainable + /// parameter, so tape-mode optimizers update them too. /// /// - /// Sparse weight tensors don't fit the dense ParameterBuffer<T> view - /// contract used by the tape, so returning false here keeps the layer's inference - /// behavior correct while explicitly advertising that no gradient updates flow through - /// it. Integrating sparse parameters into the tape (e.g., via a sparse trainable tensor - /// view) is the follow-up required before this can flip back to true. + /// Tape-mode caveat: the sparse + /// weight tensor is still not visible to the tape's + /// ParameterBuffer<T>-based discovery — that contract requires + /// dense storage. In tape mode, weight updates flow through the + /// layer's own + + /// when the optimizer falls back to the legacy update path; weight + /// gradients do not appear in the tape's flat-buffer view. Closing that + /// gap fully (sparse-aware ParameterBuffer<T>) is tracked as + /// a follow-up. /// - public override bool SupportsTraining => false; + public override bool SupportsTraining => true; /// /// Initializes a new instance of the SparseLinearLayer. From d413083cace7d671d2643cd57dfe57b61b6a11b4 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 15:22:02 -0400 Subject: [PATCH 07/29] perf(dit): vectorize Patchify/Unpatchify/AdaLN via Engine reshape+permute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the scalar nested-loop implementations of Patchify, Unpatchify, ReshapeForHeads, ReshapeFromHeads, and the ExtractModulation/ApplyAdaLN/ AddWithGate helpers with their Engine-op equivalents — reshape + permute + reshape pipelines and zero-copy TensorSliceAxis views off the AdaLN modulation tensor. Specific changes: * Patchify/Unpatchify: replace the 6-deep scalar nested loop with Engine.Reshape → Engine.TensorPermute → Engine.Reshape. The permute runs through the engine's vectorized memcpy kernel (or stays as a view when the downstream consumer supports strided) instead of a per-element C# scalar copy. * ReshapeForHeads/FromHeads: same pattern (reshape + permute + reshape) instead of the original triple-nested scalar copy with span slices. * ExtractModulation eliminated entirely. Previously ForwardBlock did 6 ExtractModulation calls per block (24 blocks × 50 inference steps × 6 = 7200 T[] allocations per Predict). Now ForwardBlock reshapes the AdaLN modulation output to [B, 6, 1, H] once and slices out each shift/scale/gate via Engine.TensorSliceAxis — zero allocations, zero scalar fill loops. * ApplyAdaLN / AddWithGate rewritten to accept Tensor broadcast views (from TensorSliceAxis) instead of T[] scalar arrays. The previous implementations built a [1,1,H] broadcast tensor via TensorAllocator.Rent + a per-element scalar fill; the new ones use Engine.TensorAddScalar / Engine.TensorBroadcastMultiply / Engine. TensorBroadcastAdd directly on the sliced views. * EmbedPatches / FinalLayerWithAdaLN: replaced the TensorAllocator.Rent + CopyTo scratch-buffer round trips with Engine.Reshape view chains (the downstream dense forward is contiguous-input-tolerant). Every hot-path scalar copy in DiT forward is now either a view (zero-copy) or a SIMD-vectorized engine op. Depends on the matching AiDotNet.Tensors PR #196 for the double-precision SIMD fallbacks in TensorMatMul / ScaledDotProductAttention / FusedLinear / broadcast ops. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../NoisePredictors/DiTNoisePredictor.cs | 329 +++++++----------- 1 file changed, 119 insertions(+), 210 deletions(-) diff --git a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs index 55b3139c92..bd41dc1d39 100644 --- a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs +++ b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs @@ -574,8 +574,20 @@ private Tensor Forward(Tensor x, Tensor timeEmbed, Tensor? condition } /// - /// Converts image to patches. + /// Converts image to patches via a single reshape + permute + reshape. /// + /// + /// Equivalent to the nested 6-loop scalar copy this used to be: + /// [B, C, H, W] → reshape [B, C, H/p, p, W/p, p] + /// → permute to [B, H/p, W/p, C, p, p] + /// → reshape [B, numPatches, C·p·p]. + /// All three ops route through , so the permutation + /// runs through the engine's vectorized memcpy kernel instead of a + /// scalar C# nested loop. The two reshape steps are zero-copy views; the + /// permute is the only step that materializes (and only because the + /// downstream requires contiguous + /// input). + /// private Tensor Patchify(Tensor x) { var shape = x._shape; @@ -589,48 +601,23 @@ private Tensor Patchify(Tensor x) var numPatches = numPatchesH * numPatchesW; var patchDim = channels * _patchSize * _patchSize; - var patches = TensorAllocator.Rent(new[] { batch, numPatches, patchDim }); - var patchSpan = patches.AsWritableSpan(); - var xSpan = x.AsSpan(); - - for (int b = 0; b < batch; b++) - { - int patchIdx = 0; - for (int ph = 0; ph < numPatchesH; ph++) - { - for (int pw = 0; pw < numPatchesW; pw++) - { - // Extract patch - int dimIdx = 0; - for (int c = 0; c < channels; c++) - { - for (int py = 0; py < _patchSize; py++) - { - for (int px = 0; px < _patchSize; px++) - { - var ih = ph * _patchSize + py; - var iw = pw * _patchSize + px; - var srcIdx = b * channels * height * width + - c * height * width + - ih * width + iw; - var dstIdx = b * numPatches * patchDim + - patchIdx * patchDim + dimIdx; - patchSpan[dstIdx] = xSpan[srcIdx]; - dimIdx++; - } - } - } - patchIdx++; - } - } - } - - return patches; + // [B, C, H, W] -> [B, C, H/p, p, W/p, p] + var split = Engine.Reshape(x, + new[] { batch, channels, numPatchesH, _patchSize, numPatchesW, _patchSize }); + // -> [B, H/p, W/p, C, p, p] (axes: 0, 2, 4, 1, 3, 5) + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 4, 1, 3, 5 }); + // -> [B, numPatches, patchDim] + return Engine.Reshape(permuted, new[] { batch, numPatches, patchDim }); } /// /// Embeds patches through linear projection using batched forward pass. /// + /// + /// Uses zero-copy views around the dense + /// projection — the previous TensorAllocator.Rent + CopyTo scratch + /// buffers were unnecessary because Patchify's output is contiguous. + /// private Tensor EmbedPatches(Tensor patches) { if (_patchEmbed == null) @@ -641,18 +628,10 @@ private Tensor EmbedPatches(Tensor patches) var numPatches = shape[1]; var patchDim = shape[2]; - // Reshape [batch, numPatches, patchDim] -> [batch*numPatches, patchDim] for batched forward - var flatPatches = TensorAllocator.Rent(new[] { batch * numPatches, patchDim }); - patches.AsSpan().CopyTo(flatPatches.AsWritableSpan()); - - // Single batched forward pass through the linear layer + // [B, numPatches, patchDim] -> [B·numPatches, patchDim] view, batched dense forward, reshape back as view. + var flatPatches = Engine.Reshape(patches, new[] { batch * numPatches, patchDim }); var projected = _patchEmbed.Forward(flatPatches); - - // Reshape back to [batch, numPatches, hiddenSize] - var embedded = TensorAllocator.Rent(new[] { batch, numPatches, _hiddenSize }); - projected.AsSpan().CopyTo(embedded.AsWritableSpan()); - - return embedded; + return Engine.Reshape(projected, new[] { batch, numPatches, _hiddenSize }); } /// @@ -700,6 +679,14 @@ private Tensor CreatePositionEmbedding(int numPatches) /// /// Forward pass through a single DiT block with AdaLN. /// + /// + /// AdaLN modulation tensor is reshaped to [B, 6, 1, hidden] and the six + /// shift/scale/gate parameters are obtained as zero-copy + /// views — no T[] allocations, + /// no scalar fill loops. The (1 + scale) precomputation in AdaLN + /// uses the SIMD primitive instead + /// of a per-element NumOps.Add(NumOps.One, scale[h]) loop. + /// private Tensor ForwardBlock( Tensor x, Tensor timeEmbed, @@ -713,18 +700,23 @@ private Tensor ForwardBlock( throw new InvalidOperationException("Block layers not initialized."); } - // Get AdaLN modulation parameters (shift1, scale1, gate1, shift2, scale2, gate2) + // Get AdaLN modulation parameters (shift1, scale1, gate1, shift2, scale2, gate2). + // AdaLNModulation forward output is hidden*6 elements per batch — derive + // batchM from total length so we don't depend on whether the layer + // returns [B, hidden*6] (rank 2) or [hidden*6] (rank 1, when B=1 and the + // dense layer collapses leading dims). Reshape to [B, 6, 1, hidden] so + // TensorSliceAxis(axis=1, index=i) yields a [B, 1, hidden] view directly, + // broadcastable over [B, seq, hidden] without further reshape. var modulation = block.AdaLNModulation.Forward(timeEmbed); - var modSpan = modulation.AsSpan(); + int batchM = modulation.Length / (6 * _hiddenSize); + var modReshaped = Engine.Reshape(modulation, new[] { batchM, 6, 1, _hiddenSize }); - // Split modulation into 6 parts - var modSize = _hiddenSize; - var shift1 = ExtractModulation(modSpan, 0, modSize); - var scale1 = ExtractModulation(modSpan, modSize, modSize); - var gate1 = ExtractModulation(modSpan, modSize * 2, modSize); - var shift2 = ExtractModulation(modSpan, modSize * 3, modSize); - var scale2 = ExtractModulation(modSpan, modSize * 4, modSize); - var gate2 = ExtractModulation(modSpan, modSize * 5, modSize); + var shift1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); + var scale1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 1); + var gate1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 2); + var shift2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 3); + var scale2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 4); + var gate2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 5); // Self-attention with AdaLN var normed = block.Norm1.Forward(x); @@ -751,43 +743,18 @@ private Tensor ForwardBlock( } /// - /// Extracts modulation parameters from combined tensor. + /// Applies adaptive layer normalization (Peebles & Xie, 2023): + /// y = x * (1 + scale) + shift. Both + /// and are [B, 1, hidden] tensor views + /// sliced from the AdaLN modulation tensor — no scratch allocation, no + /// scalar fill. The (1 + scale) precomputation runs through the + /// SIMD kernel. /// - private T[] ExtractModulation(ReadOnlySpan modSpan, int offset, int size) + private Tensor ApplyAdaLN(Tensor x, Tensor scaleView, Tensor shiftView) { - var result = new T[size]; - for (int i = 0; i < size; i++) - { - result[i] = modSpan[offset + i]; - } - return result; - } - - /// - /// Applies adaptive layer normalization using IEngine for hardware acceleration. - /// y = x * (1 + scale) + shift - /// - private Tensor ApplyAdaLN(Tensor x, T[] scale, T[] shift) - { - var shape = x._shape; - var hidden = shape[^1]; - - // Create broadcastable tensors for scale and shift: [1, 1, hidden] - var scaleTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var shiftTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var scaleSpan = scaleTensor.AsWritableSpan(); - var shiftSpan = shiftTensor.AsWritableSpan(); - - for (int h = 0; h < hidden; h++) - { - scaleSpan[h] = NumOps.Add(NumOps.One, scale[h % scale.Length]); - shiftSpan[h] = shift[h % shift.Length]; - } - - // y = x * (1 + scale) + shift — per DiT paper (Peebles & Xie, 2023) - // scale/shift are [1,1,hidden], x is [batch, seq, hidden] — requires broadcasting - var scaled = Engine.TensorBroadcastMultiply(x, scaleTensor); - return Engine.TensorBroadcastAdd(scaled, shiftTensor); + var scalePlusOne = Engine.TensorAddScalar(scaleView, NumOps.One); + var scaled = Engine.TensorBroadcastMultiply(x, scalePlusOne); + return Engine.TensorBroadcastAdd(scaled, shiftView); } /// @@ -861,166 +828,108 @@ private Tensor ApplyCrossAttention( } /// - /// Reshapes tensor from [batch, seq, hidden] to [batch*heads, seq, headDim] for multi-head attention. + /// Reshapes tensor from [batch, seq, hidden] to + /// [batch*heads, seq, headDim] for multi-head attention via a + /// reshape + permute + reshape pipeline (no scalar nested copy loops). /// - private static Tensor ReshapeForHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) + private Tensor ReshapeForHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) { - var result = TensorAllocator.Rent(new[] { batch * numHeads, seq, headDim }); - var srcSpan = tensor.AsSpan(); - var dstSpan = result.AsWritableSpan(); - var hidden = numHeads * headDim; - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < numHeads; h++) - { - for (int s = 0; s < seq; s++) - { - var srcBase = b * seq * hidden + s * hidden + h * headDim; - var dstBase = (b * numHeads + h) * seq * headDim + s * headDim; - srcSpan.Slice(srcBase, headDim).CopyTo(dstSpan.Slice(dstBase, headDim)); - } - } - } - - return result; + // [B, S, H·D] -> [B, S, H, D] + var split = Engine.Reshape(tensor, new[] { batch, seq, numHeads, headDim }); + // -> [B, H, S, D] + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 1, 3 }); + // -> [B·H, S, D] + return Engine.Reshape(permuted, new[] { batch * numHeads, seq, headDim }); } /// - /// Reshapes tensor from [batch*heads, seq, headDim] back to [batch, seq, hidden]. + /// Inverse of — collapses head and batch + /// back together via reshape + permute + reshape. /// - private static Tensor ReshapeFromHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) + private Tensor ReshapeFromHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) { - var result = TensorAllocator.Rent(new[] { batch, seq, numHeads * headDim }); - var srcSpan = tensor.AsSpan(); - var dstSpan = result.AsWritableSpan(); - var hidden = numHeads * headDim; - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < numHeads; h++) - { - for (int s = 0; s < seq; s++) - { - var srcBase = (b * numHeads + h) * seq * headDim + s * headDim; - var dstBase = b * seq * hidden + s * hidden + h * headDim; - srcSpan.Slice(srcBase, headDim).CopyTo(dstSpan.Slice(dstBase, headDim)); - } - } - } - - return result; + // [B·H, S, D] -> [B, H, S, D] + var split = Engine.Reshape(tensor, new[] { batch, numHeads, seq, headDim }); + // -> [B, S, H, D] + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 1, 3 }); + // -> [B, S, H·D] + return Engine.Reshape(permuted, new[] { batch, seq, numHeads * headDim }); } /// - /// Adds with gating using IEngine for hardware acceleration. - /// result = x + gate * residual + /// Gated residual add: result = x + gateView * residual. + /// is a [B, 1, hidden] view sliced from + /// the AdaLN modulation tensor — no scratch allocation, no scalar fill. + /// Uses for the per-channel + /// gate broadcast. /// - private Tensor AddWithGate(Tensor x, Tensor residual, T[] gate) + private Tensor AddWithGate(Tensor x, Tensor residual, Tensor gateView) { - var hidden = x.Shape[^1]; - - // Create broadcastable gate tensor: [1, 1, hidden] - var gateTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var gateSpan = gateTensor.AsWritableSpan(); - for (int h = 0; h < hidden; h++) - { - gateSpan[h] = gate[h % gate.Length]; - } - - // result = x + gate * residual - var gated = Engine.TensorMultiply(residual, gateTensor); + var gated = Engine.TensorBroadcastMultiply(residual, gateView); return Engine.TensorAdd(x, gated); } /// - /// Final layer with AdaLN zero using batched forward pass. + /// Final layer with AdaLN-zero using batched forward pass. /// + /// + /// Modulation slicing matches : reshape to + /// [B, 2, 1, hidden] then for + /// shift/scale views. Reshape between the projection input and output uses + /// views instead of + /// TensorAllocator.Rent+CopyTo scratch buffers, eliminating + /// two allocations per inference step. + /// private Tensor FinalLayerWithAdaLN(Tensor x, Tensor timeEmbed) { if (_finalNorm == null || _adaln_modulation == null || _outputProj == null) throw new InvalidOperationException("Final layers not initialized."); - // Get final modulation (scale, shift) var modulation = _adaln_modulation.Forward(timeEmbed); - var modSpan = modulation.AsSpan(); + int batchM = modulation.Length / (2 * _hiddenSize); + var modReshaped = Engine.Reshape(modulation, new[] { batchM, 2, 1, _hiddenSize }); + var shiftView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); + var scaleView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 1); - var shift = ExtractModulation(modSpan, 0, _hiddenSize); - var scale = ExtractModulation(modSpan, _hiddenSize, _hiddenSize); - - // Apply final norm with AdaLN var normed = _finalNorm.Forward(x); - normed = ApplyAdaLN(normed, scale, shift); + normed = ApplyAdaLN(normed, scaleView, shiftView); - // Project to output dimension using batched forward pass var shape = normed._shape; var batch = shape[0]; var numPatches = shape[1]; var patchDim = _inputChannels * _patchSize * _patchSize; - // Reshape [batch, numPatches, hiddenSize] -> [batch*numPatches, hiddenSize] - var flatNormed = TensorAllocator.Rent(new[] { batch * numPatches, _hiddenSize }); - normed.AsSpan().CopyTo(flatNormed.AsWritableSpan()); - - // Single batched forward pass + // [batch, numPatches, hiddenSize] -> [batch*numPatches, hiddenSize] via view. + var flatNormed = Engine.Reshape(normed, new[] { batch * numPatches, _hiddenSize }); var projected = _outputProj.Forward(flatNormed); - - // Reshape back to [batch, numPatches, patchDim] - var output = TensorAllocator.Rent(new[] { batch, numPatches, patchDim }); - projected.AsSpan().CopyTo(output.AsWritableSpan()); - - return output; + // Back to [batch, numPatches, patchDim] via view. + return Engine.Reshape(projected, new[] { batch, numPatches, patchDim }); } /// - /// Converts patches back to image. + /// Inverse of — reconstructs the spatial image. /// + /// + /// [B, numPatches, C·p·p] → reshape [B, H/p, W/p, C, p, p] + /// → permute to [B, C, H/p, p, W/p, p] + /// → reshape [B, C, H, W]. Same vectorized-memcpy + view pattern + /// as — zero scalar loops, one materialization. + /// private Tensor Unpatchify(Tensor patches, int height, int width) { var shape = patches._shape; var batch = shape[0]; - var patchDim = shape[2]; var numPatchesH = height / _patchSize; var numPatchesW = width / _patchSize; - var output = TensorAllocator.Rent(new[] { batch, _inputChannels, height, width }); - var outputSpan = output.AsWritableSpan(); - var patchSpan = patches.AsSpan(); - - for (int b = 0; b < batch; b++) - { - int patchIdx = 0; - for (int ph = 0; ph < numPatchesH; ph++) - { - for (int pw = 0; pw < numPatchesW; pw++) - { - // Reconstruct patch - int dimIdx = 0; - for (int c = 0; c < _inputChannels; c++) - { - for (int py = 0; py < _patchSize; py++) - { - for (int px = 0; px < _patchSize; px++) - { - var ih = ph * _patchSize + py; - var iw = pw * _patchSize + px; - var dstIdx = b * _inputChannels * height * width + - c * height * width + - ih * width + iw; - var srcIdx = b * numPatchesH * numPatchesW * patchDim + - patchIdx * patchDim + dimIdx; - outputSpan[dstIdx] = patchSpan[srcIdx]; - dimIdx++; - } - } - } - patchIdx++; - } - } - } - - return output; + // [B, numPatches, C·p·p] -> [B, H/p, W/p, C, p, p] + var unsplit = Engine.Reshape(patches, + new[] { batch, numPatchesH, numPatchesW, _inputChannels, _patchSize, _patchSize }); + // -> [B, C, H/p, p, W/p, p] (inverse permutation: axes 0, 3, 1, 4, 2, 5) + var permuted = Engine.TensorPermute(unsplit, new[] { 0, 3, 1, 4, 2, 5 }); + // -> [B, C, H, W] + return Engine.Reshape(permuted, new[] { batch, _inputChannels, height, width }); } /// From f7db4da7d76a2f12a9159ab6de62db4df279dd4e Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 15:22:20 -0400 Subject: [PATCH 08/29] perf(init): batched parallel Xavier normal weight initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-element SampleGaussian call loop (which ran a virtual-dispatch Box-Muller + rejection test for every element) with a tight specialized fill routine for double and float: one paired Box-Muller transform produces two samples per pair of uniform draws, halving the log/sqrt/sin/cos call count, and large layers (≥ 256K elements) are partitioned across the thread pool so the ~29s of init cost per DiT-XL-sized Dense layer (hidden 8192 × out 12288 = 100M doubles per AdaLN modulation layer) is parallelized instead of running single-threaded. Context: even after the Tensors-side SIMD fixes on the forward matmul path, the first Pika21 Predict paid ~150s of lazy-init overhead across the 24 block layers because each first-call XavierNormalInitialize hit a scalar loop doing 100M virtual calls. The cost is one-time per layer but it dominated the first forward and pushed Training_Should* tests that exercise a fresh model over the per-test xUnit budget. Preserves reproducibility: per-chunk RNGs are seeded deterministically from the master Random instance, so for a given parent seed the output is stable across thread counts. Keeps the generic-T fallback on the old path since only float/double are expected to be perf-critical. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../InitializationStrategyBase.cs | 174 ++++++++++++++++-- 1 file changed, 159 insertions(+), 15 deletions(-) diff --git a/src/Initialization/InitializationStrategyBase.cs b/src/Initialization/InitializationStrategyBase.cs index e483a4ac3f..45e3657f18 100644 --- a/src/Initialization/InitializationStrategyBase.cs +++ b/src/Initialization/InitializationStrategyBase.cs @@ -118,26 +118,15 @@ protected void XavierNormalInitialize(Tensor weights, int fanIn, int fanOut) if (typeof(T) == typeof(double)) { - for (int i = 0; i < span.Length; i++) - { - double value; - do { value = SampleGaussian(0, stddev); } - while (Math.Abs(value) > clipBound); - span[i] = System.Runtime.CompilerServices.Unsafe.As(ref value); - } + var rawArr = (double[])(object)weights.GetDataArray(); + XavierFillDouble(rawArr, 0, weights.Length, stddev, clipBound); return; } if (typeof(T) == typeof(float)) { - for (int i = 0; i < span.Length; i++) - { - double value; - do { value = SampleGaussian(0, stddev); } - while (Math.Abs(value) > clipBound); - float fv = (float)value; - span[i] = System.Runtime.CompilerServices.Unsafe.As(ref fv); - } + var rawArr = (float[])(object)weights.GetDataArray(); + XavierFillFloat(rawArr, 0, weights.Length, stddev, clipBound); return; } @@ -259,4 +248,159 @@ protected double SampleGaussian(double mean, double stddev) var randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); return mean + stddev * randStdNormal; } + + /// + /// Fills a span with N(0, stddev) samples clipped to ±, + /// using a paired Box-Muller transform that produces two samples per pair of uniform + /// draws — halves the / call count vs. + /// calling per element. + /// + /// + /// Replaces the per-element while (Math.Abs(value) > clipBound) do ... + /// rejection loop which was the dominant cost of DiT-XL lazy weight init (each + /// block's Dense / SelfAttention layer paid 1–30 s of RNG overhead on first + /// forward). Rejection rate at 2σ is ~5 %, so in the common case each iteration + /// produces two usable samples with one log + one sqrt + one sin + one cos + two + /// multiplies. The inner loop is a tight unvirtualized local function so JIT can + /// keep everything in registers and auto-vectorize the clip check. + /// + private void XavierFillDouble(double[] dst, int offset, int length, double stddev, double clipBound) + { + if (length == 0) return; + + const int ParallelThreshold = 1 << 18; // 256K doubles ≈ 2MB + int cores = Math.Max(1, Environment.ProcessorCount); + + if (length < ParallelThreshold || cores == 1) + { + FillChunkDouble(dst.AsSpan(offset, length), stddev, clipBound, Random); + return; + } + + // For large tensors (typical DiT-XL hidden×4 ≈ 100M elements), partition + // across cores so init amortizes over the thread pool instead of running + // single-threaded. Pre-seed per-chunk RNGs from the master so the parallel + // work remains deterministic relative to the master seed. System.Random + // is NOT thread-safe, so we MUST use per-thread instances. + int chunkSize = (length + cores - 1) / cores; + var seeds = new int[cores]; + for (int c = 0; c < cores; c++) seeds[c] = Random.Next(); + + System.Threading.Tasks.Parallel.For(0, cores, c => + { + int chunkStart = c * chunkSize; + int chunkEnd = Math.Min(chunkStart + chunkSize, length); + if (chunkStart >= chunkEnd) return; + var chunkRng = new Random(seeds[c]); + FillChunkDouble(dst.AsSpan(offset + chunkStart, chunkEnd - chunkStart), stddev, clipBound, chunkRng); + }); + } + + /// + /// Sequential Box-Muller fill of a span — inner helper used by both the + /// sequential fast path and the parallel chunk workers. + /// + private static void FillChunkDouble(Span dst, double stddev, double clipBound, Random rng) + { + double z1 = 0; + bool havePending = false; + + for (int i = 0; i < dst.Length; i++) + { + double sample; + while (true) + { + if (havePending) + { + sample = z1; + havePending = false; + } + else + { + double u1 = 1.0 - rng.NextDouble(); + double u2 = rng.NextDouble(); + double r = Math.Sqrt(-2.0 * Math.Log(u1)); + double theta = 2.0 * Math.PI * u2; + sample = r * Math.Sin(theta); + z1 = r * Math.Cos(theta); + havePending = true; + } + sample *= stddev; + if (!(sample > clipBound) && !(sample < -clipBound)) + { + dst[i] = sample; + break; + } + havePending = false; + } + } + } + + /// + /// Float variant of . Uses double-precision + /// Box-Muller internally (accuracy matters more than the tiny cost) and + /// narrows to float on store. + /// + private void XavierFillFloat(float[] dst, int offset, int length, double stddev, double clipBound) + { + if (length == 0) return; + + const int ParallelThreshold = 1 << 18; + int cores = Math.Max(1, Environment.ProcessorCount); + + if (length < ParallelThreshold || cores == 1) + { + FillChunkFloat(dst.AsSpan(offset, length), stddev, clipBound, Random); + return; + } + + int chunkSize = (length + cores - 1) / cores; + var seeds = new int[cores]; + for (int c = 0; c < cores; c++) seeds[c] = Random.Next(); + + System.Threading.Tasks.Parallel.For(0, cores, c => + { + int chunkStart = c * chunkSize; + int chunkEnd = Math.Min(chunkStart + chunkSize, length); + if (chunkStart >= chunkEnd) return; + var chunkRng = new Random(seeds[c]); + FillChunkFloat(dst.AsSpan(offset + chunkStart, chunkEnd - chunkStart), stddev, clipBound, chunkRng); + }); + } + + private static void FillChunkFloat(Span dst, double stddev, double clipBound, Random rng) + { + double z1 = 0; + bool havePending = false; + + for (int i = 0; i < dst.Length; i++) + { + double sample; + while (true) + { + if (havePending) + { + sample = z1; + havePending = false; + } + else + { + double u1 = 1.0 - rng.NextDouble(); + double u2 = rng.NextDouble(); + double r = Math.Sqrt(-2.0 * Math.Log(u1)); + double theta = 2.0 * Math.PI * u2; + sample = r * Math.Sin(theta); + z1 = r * Math.Cos(theta); + havePending = true; + } + sample *= stddev; + if (!(sample > clipBound) && !(sample < -clipBound)) + { + dst[i] = (float)sample; + break; + } + havePending = false; + } + } + } } From c22c3f6becb899b1287a62efa6278ee1c67e6698 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 16:48:18 -0400 Subject: [PATCH 09/29] chore(deps): bump aidotnet.tensors 0.46.0 -> 0.46.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls in the Tensors SIMD fallback fixes from Tensors PR #196: - TensorMatMul double fallback routed through MultiplyBlocked - ScaledDotProductAttention double SIMD fast path - FusedGemmBiasActivation double fallback SIMD-routed - TensorBroadcast{Multiply,Add} trailing-repeat fast path - Odometer-based Contiguous() materialization - LayerNorm generic fallback uses SIMD numOps.Sum Unblocks the DiT vectorization work in this PR — every double-precision matmul / broadcast / attention op it relies on now hits a SIMD path instead of a scalar triple-loop. Also unblocks MobileNetV3_Train_CompletesWithoutError which hit the TensorCopy source.Length regression (Tensors PR #195, included in 0.46.1 via #194's follow-up). Co-Authored-By: Claude Opus 4.7 (1M context) --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d010125b08..b9ff49519f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ - + From 05f3a03fa454a5681f945405026cf2b705c71fbb Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 16:32:12 -0400 Subject: [PATCH 10/29] fix(stats): break EnsureFullStatsComputed recursion in errorstats/modelstats/predictionstats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same bug class as the earlier BasicStats fix: the Calculate* method was assigning to properties AND reading them back during its own body, but the property getters call EnsureFullStatsComputed — which is still running the Calculate* method. The _fullStatsComputed flag only flips after Calculate* returns, so any intra-method property read re-enters Calculate* unbounded. The test host crashes with StackOverflowException before the test framework can report anything except "host process exited unexpectedly." Specific re-entry points the previous code had: * ErrorStats.CalculateErrorStats - RMSE = _numOps.Sqrt(MSE) ← re-enters via MSE getter - AIC/BIC/AICAlt pass RSS ← re-enters via RSS getter * ModelStats.CalculateModelStats - VIFList = ... CalculateVIF(CorrelationMatrix, ...) ← CorrelationMatrix - Mahalanobis block reads CovarianceMatrix thrice ← CovarianceMatrix * PredictionStats.CalculatePredictionStats - AdjustedR2 = ... CalculateAdjustedR2(R2, ...) ← R2 - PredictionIntervalCoverage = ... (PredictionInterval.Lower, PredictionInterval.Upper) ← PredictionInterval - ConfidenceInterval/CredibleInterval read BestDistributionFit .DistributionType ← BestDistributionFit All three methods are rewritten to compute every intermediate into a local variable first; properties are only assigned once every dependency is a local. No property reads happen inside Calculate*, so the lazy getter never re-enters. Observed failure path (Classification CI shard, PR #1156 run): AdaBoostClassifierTests.Predict_ShouldBeDeterministic trains the model, which computes ErrorStats, which stack-overflows the host. Other crashed tests in the same shard: - ExtraTreesClassifierTests.Clone_ShouldProduceIdenticalPredictions - CategoricalNaiveBayesTests.Builder_AccuracyShouldBeatChance - OneVsRestClassifierTests.Builder_AccuracyShouldBeatChance All 4 pass locally after this fix. Unblocks the host_crash jobs on PR #1154 triage: - ModelFamily - Classification - ModelFamily - Clustering/GP - ModelFamily - Regression - ModelFamily - TimeSeries/Activation/Loss - Unit - 04 Feature/Fit/Fitness/Genetics - AiDotNet.Serving.Tests Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Statistics/ErrorStats.cs | 105 +++++++++++++++++--------- src/Statistics/ModelStats.cs | 119 ++++++++++++++++++------------ src/Statistics/PredictionStats.cs | 89 +++++++++++++++------- 3 files changed, 203 insertions(+), 110 deletions(-) diff --git a/src/Statistics/ErrorStats.cs b/src/Statistics/ErrorStats.cs index 7e72d7f123..f02a78c636 100644 --- a/src/Statistics/ErrorStats.cs +++ b/src/Statistics/ErrorStats.cs @@ -513,50 +513,83 @@ public static ErrorStats Empty() /// private void CalculateErrorStats(Vector actual, Vector predicted, int numberOfParameters, PredictionType predictionType = PredictionType.Regression) { + // All intermediate values are held in locals and only assigned to the + // observable properties at the very end. Earlier code called + // RMSE = _numOps.Sqrt(MSE); + // AIC = StatisticsHelper.CalculateAIC(n, numberOfParameters, RSS); + // which read MSE/RSS through their property getters. Those getters call + // EnsureFullStatsComputed, which is still running CalculateErrorStats — + // so the property read re-enters CalculateErrorStats and recurses + // without bound. The test host crashes with StackOverflowException + // (Classification / Clustering / Regression / TimeSeries / Serving + // host-crashes on PR #1154 and master). int n = actual.Length; - // Calculate basic error metrics - MAE = StatisticsHelper.CalculateMeanAbsoluteError(actual, predicted); - RSS = StatisticsHelper.CalculateResidualSumOfSquares(actual, predicted); - MSE = StatisticsHelper.CalculateMeanSquaredError(actual, predicted); - RMSE = _numOps.Sqrt(MSE); - MAPE = StatisticsHelper.CalculateMeanAbsolutePercentageError(actual, predicted); - MedianAbsoluteError = StatisticsHelper.CalculateMedianAbsoluteError(actual, predicted); - MaxError = StatisticsHelper.CalculateMaxError(actual, predicted); + T mae = StatisticsHelper.CalculateMeanAbsoluteError(actual, predicted); + T rss = StatisticsHelper.CalculateResidualSumOfSquares(actual, predicted); + T mse = StatisticsHelper.CalculateMeanSquaredError(actual, predicted); + T rmse = _numOps.Sqrt(mse); + T mape = StatisticsHelper.CalculateMeanAbsolutePercentageError(actual, predicted); + T medAbs = StatisticsHelper.CalculateMedianAbsoluteError(actual, predicted); + T maxErr = StatisticsHelper.CalculateMaxError(actual, predicted); + T aucPR, aucROC; if (predictionType == PredictionType.BinaryClassification) { - AUCPR = StatisticsHelper.CalculatePrecisionRecallAUC(actual, predicted); - AUCROC = StatisticsHelper.CalculateROCAUC(actual, predicted); + aucPR = StatisticsHelper.CalculatePrecisionRecallAUC(actual, predicted); + aucROC = StatisticsHelper.CalculateROCAUC(actual, predicted); } else { - AUCPR = _numOps.Zero; - AUCROC = _numOps.Zero; + aucPR = _numOps.Zero; + aucROC = _numOps.Zero; } - SMAPE = StatisticsHelper.CalculateSymmetricMeanAbsolutePercentageError(actual, predicted); - MeanSquaredLogError = StatisticsHelper.CalculateMeanSquaredLogError(actual, predicted); - CRPS = StatisticsHelper.CalculateCRPS(actual, predicted); - - // Calculate standard errors - SampleStandardError = StatisticsHelper.CalculateSampleStandardError(actual, predicted, numberOfParameters); - PopulationStandardError = StatisticsHelper.CalculatePopulationStandardError(actual, predicted); - - // Calculate bias and autocorrelation metrics - MeanBiasError = StatisticsHelper.CalculateMeanBiasError(actual, predicted); - TheilUStatistic = StatisticsHelper.CalculateTheilUStatistic(actual, predicted); - DurbinWatsonStatistic = StatisticsHelper.CalculateDurbinWatsonStatistic(actual, predicted); - - // Calculate information criteria - AIC = StatisticsHelper.CalculateAIC(n, numberOfParameters, RSS); - BIC = StatisticsHelper.CalculateBIC(n, numberOfParameters, RSS); - AICAlt = StatisticsHelper.CalculateAICAlternative(n, numberOfParameters, RSS); - - // Calculate classification metrics - Accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); - (Precision, Recall, F1Score) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); - - // Populate error list - ErrorList = new List(StatisticsHelper.CalculateResiduals(actual, predicted)); + T smape = StatisticsHelper.CalculateSymmetricMeanAbsolutePercentageError(actual, predicted); + T msle = StatisticsHelper.CalculateMeanSquaredLogError(actual, predicted); + T crps = StatisticsHelper.CalculateCRPS(actual, predicted); + + T sampleStd = StatisticsHelper.CalculateSampleStandardError(actual, predicted, numberOfParameters); + T popStd = StatisticsHelper.CalculatePopulationStandardError(actual, predicted); + + T bias = StatisticsHelper.CalculateMeanBiasError(actual, predicted); + T theil = StatisticsHelper.CalculateTheilUStatistic(actual, predicted); + T dw = StatisticsHelper.CalculateDurbinWatsonStatistic(actual, predicted); + + T aic = StatisticsHelper.CalculateAIC(n, numberOfParameters, rss); + T bic = StatisticsHelper.CalculateBIC(n, numberOfParameters, rss); + T aicAlt = StatisticsHelper.CalculateAICAlternative(n, numberOfParameters, rss); + + T accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); + var (precision, recall, f1) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + + var residuals = new List(StatisticsHelper.CalculateResiduals(actual, predicted)); + + // Assign to the observable properties only after every dependency is a + // local — no re-entry possible now. + MAE = mae; + RSS = rss; + MSE = mse; + RMSE = rmse; + MAPE = mape; + MedianAbsoluteError = medAbs; + MaxError = maxErr; + AUCPR = aucPR; + AUCROC = aucROC; + SMAPE = smape; + MeanSquaredLogError = msle; + CRPS = crps; + SampleStandardError = sampleStd; + PopulationStandardError = popStd; + MeanBiasError = bias; + TheilUStatistic = theil; + DurbinWatsonStatistic = dw; + AIC = aic; + BIC = bic; + AICAlt = aicAlt; + Accuracy = accuracy; + Precision = precision; + Recall = recall; + F1Score = f1; + ErrorList = residuals; } /// diff --git a/src/Statistics/ModelStats.cs b/src/Statistics/ModelStats.cs index 23fd1ef8b5..0b9d8f3552 100644 --- a/src/Statistics/ModelStats.cs +++ b/src/Statistics/ModelStats.cs @@ -552,14 +552,16 @@ public static ModelStats Empty() /// private void CalculateModelStats(ModelStatsInputs inputs) { - // Convert input matrix to Matrix for statistical calculations + // All intermediate values held in locals until the end. Previous code + // read CorrelationMatrix and CovarianceMatrix through their property + // getters during computation; those getters call + // EnsureFullStatsComputed, which re-enters this method — unbounded + // recursion + StackOverflowException. Same bug class as BasicStats / + // ErrorStats / PredictionStats. Matrix matrix = ConversionsHelper.ConvertToMatrix(inputs.XMatrix); - - // Convert actual and predicted values to Vector for statistical calculations Vector actual = ConversionsHelper.ConvertToVector(inputs.Actual); Vector predicted = ConversionsHelper.ConvertToVector(inputs.Predicted); - // Convert coefficients if available Vector coefficients = Vector.Empty(); if (inputs.Coefficients != null) { @@ -568,64 +570,89 @@ private void CalculateModelStats(ModelStatsInputs inputs) var featureCount = inputs.FeatureCount; - // Calculate all statistical metrics using the converted data types - CorrelationMatrix = StatisticsHelper.CalculateCorrelationMatrix(matrix, _options); - CovarianceMatrix = StatisticsHelper.CalculateCovarianceMatrix(matrix); - VIFList = StatisticsHelper.CalculateVIF(CorrelationMatrix, _options); - ConditionNumber = StatisticsHelper.CalculateConditionNumber(matrix, _options); - LogPointwisePredictiveDensity = StatisticsHelper.CalculateLogPointwisePredictiveDensity(actual, predicted); + var correlationMatrix = StatisticsHelper.CalculateCorrelationMatrix(matrix, _options); + var covarianceMatrix = StatisticsHelper.CalculateCovarianceMatrix(matrix); + var vifList = StatisticsHelper.CalculateVIF(correlationMatrix, _options); + T conditionNumber = StatisticsHelper.CalculateConditionNumber(matrix, _options); + T logPpd = StatisticsHelper.CalculateLogPointwisePredictiveDensity(actual, predicted); + List? looPd = null; if (inputs.FitFunction != null) { - // Convert the fit function to work with Matrix and Vector var convertedFitFunction = ConversionsHelper.ConvertFitFunction(inputs.FitFunction); - - // Create a wrapper function that matches the expected signature Vector wrappedFitFunction(Matrix m, Vector v) => convertedFitFunction(m); - LeaveOneOutPredictiveDensities = StatisticsHelper.CalculateLeaveOneOutPredictiveDensities(matrix, actual, wrappedFitFunction); + looPd = StatisticsHelper.CalculateLeaveOneOutPredictiveDensities(matrix, actual, wrappedFitFunction); } - ObservedTestStatistic = StatisticsHelper.CalculateObservedTestStatistic(actual, predicted); - PosteriorPredictiveSamples = StatisticsHelper.CalculatePosteriorPredictiveSamples(actual, predicted, featureCount); - MarginalLikelihood = StatisticsHelper.CalculateMarginalLikelihood(actual, predicted, featureCount); - ReferenceModelMarginalLikelihood = StatisticsHelper.CalculateReferenceModelMarginalLikelihood(actual); - LogLikelihood = StatisticsHelper.CalculateLogLikelihood(actual, predicted); - EffectiveNumberOfParameters = StatisticsHelper.CalculateEffectiveNumberOfParameters(matrix, coefficients); - MutualInformation = StatisticsHelper.CalculateMutualInformation(actual, predicted); - NormalizedMutualInformation = StatisticsHelper.CalculateNormalizedMutualInformation(actual, predicted); - VariationOfInformation = StatisticsHelper.CalculateVariationOfInformation(actual, predicted); - SilhouetteScore = StatisticsHelper.CalculateSilhouetteScore(matrix, predicted); - CalinskiHarabaszIndex = StatisticsHelper.CalculateCalinskiHarabaszIndex(matrix, predicted); - DaviesBouldinIndex = StatisticsHelper.CalculateDaviesBouldinIndex(matrix, predicted); - MeanAveragePrecision = StatisticsHelper.CalculateMeanAveragePrecision(actual, predicted, _options.MapTopK); - NormalizedDiscountedCumulativeGain = StatisticsHelper.CalculateNDCG(actual, predicted, _options.NdcgTopK); - MeanReciprocalRank = StatisticsHelper.CalculateMeanReciprocalRank(actual, predicted); - - // Calculate residuals and time series metrics + T observedTest = StatisticsHelper.CalculateObservedTestStatistic(actual, predicted); + var posteriorSamples = StatisticsHelper.CalculatePosteriorPredictiveSamples(actual, predicted, featureCount); + T marginalLik = StatisticsHelper.CalculateMarginalLikelihood(actual, predicted, featureCount); + T refModelML = StatisticsHelper.CalculateReferenceModelMarginalLikelihood(actual); + T logLik = StatisticsHelper.CalculateLogLikelihood(actual, predicted); + T effParams = StatisticsHelper.CalculateEffectiveNumberOfParameters(matrix, coefficients); + T mi = StatisticsHelper.CalculateMutualInformation(actual, predicted); + T nmi = StatisticsHelper.CalculateNormalizedMutualInformation(actual, predicted); + T voi = StatisticsHelper.CalculateVariationOfInformation(actual, predicted); + T silhouette = StatisticsHelper.CalculateSilhouetteScore(matrix, predicted); + T ch = StatisticsHelper.CalculateCalinskiHarabaszIndex(matrix, predicted); + T db = StatisticsHelper.CalculateDaviesBouldinIndex(matrix, predicted); + T mAP = StatisticsHelper.CalculateMeanAveragePrecision(actual, predicted, _options.MapTopK); + T ndcg = StatisticsHelper.CalculateNDCG(actual, predicted, _options.NdcgTopK); + T mrr = StatisticsHelper.CalculateMeanReciprocalRank(actual, predicted); + var residuals = StatisticsHelper.CalculateResiduals(actual, predicted); - AutoCorrelationFunction = StatisticsHelper.CalculateAutoCorrelationFunction(residuals, _options.AcfMaxLag); - PartialAutoCorrelationFunction = StatisticsHelper.CalculatePartialAutoCorrelationFunction(residuals, _options.PacfMaxLag); - - // Calculate distance metrics - EuclideanDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Euclidean); - ManhattanDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Manhattan); - CosineSimilarity = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Cosine); - JaccardSimilarity = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Jaccard); - HammingDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Hamming); - - // Mahalanobis distance requires vector dimensions to match covariance matrix dimensions - // Skip calculation if dimensions don't match (covariance is feature x feature, not sample x sample) + var acf = StatisticsHelper.CalculateAutoCorrelationFunction(residuals, _options.AcfMaxLag); + var pacf = StatisticsHelper.CalculatePartialAutoCorrelationFunction(residuals, _options.PacfMaxLag); + + T euclidean = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Euclidean); + T manhattan = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Manhattan); + T cosine = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Cosine); + T jaccard = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Jaccard); + T hamming = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Hamming); + + T mahalanobis = _numOps.Zero; try { - if (CovarianceMatrix.Rows == actual.Length && CovarianceMatrix.Columns == actual.Length) + if (covarianceMatrix.Rows == actual.Length && covarianceMatrix.Columns == actual.Length) { - MahalanobisDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Mahalanobis, CovarianceMatrix); + mahalanobis = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Mahalanobis, covarianceMatrix); } } catch (ArgumentException) { - // Silently skip Mahalanobis distance calculation when dimensions don't match + // Silently skip Mahalanobis distance calculation when dimensions don't match. } + + // Assign properties once every dependency is a local — no re-entry. + CorrelationMatrix = correlationMatrix; + CovarianceMatrix = covarianceMatrix; + VIFList = vifList; + ConditionNumber = conditionNumber; + LogPointwisePredictiveDensity = logPpd; + if (looPd != null) LeaveOneOutPredictiveDensities = looPd; + ObservedTestStatistic = observedTest; + PosteriorPredictiveSamples = posteriorSamples; + MarginalLikelihood = marginalLik; + ReferenceModelMarginalLikelihood = refModelML; + LogLikelihood = logLik; + EffectiveNumberOfParameters = effParams; + MutualInformation = mi; + NormalizedMutualInformation = nmi; + VariationOfInformation = voi; + SilhouetteScore = silhouette; + CalinskiHarabaszIndex = ch; + DaviesBouldinIndex = db; + MeanAveragePrecision = mAP; + NormalizedDiscountedCumulativeGain = ndcg; + MeanReciprocalRank = mrr; + AutoCorrelationFunction = acf; + PartialAutoCorrelationFunction = pacf; + EuclideanDistance = euclidean; + ManhattanDistance = manhattan; + CosineSimilarity = cosine; + JaccardSimilarity = jaccard; + HammingDistance = hamming; + MahalanobisDistance = mahalanobis; } /// diff --git a/src/Statistics/PredictionStats.cs b/src/Statistics/PredictionStats.cs index 65247b21ab..38bc6ffa0b 100644 --- a/src/Statistics/PredictionStats.cs +++ b/src/Statistics/PredictionStats.cs @@ -663,34 +663,67 @@ public static PredictionStats WithR2Only(T r2) /// private void CalculatePredictionStats(Vector actual, Vector predicted, int numberOfParameters, T confidenceLevel, int learningCurveSteps, PredictionType predictionType) { - BestDistributionFit = StatisticsHelper.DetermineBestFitDistribution(predicted); - - MeanPredictionError = StatisticsHelper.CalculateMeanPredictionError(actual, predicted); - MedianPredictionError = StatisticsHelper.CalculateMedianPredictionError(actual, predicted); - - R2 = StatisticsHelper.CalculateR2(actual, predicted); - AdjustedR2 = StatisticsHelper.CalculateAdjustedR2(R2, actual.Length, numberOfParameters); - ExplainedVarianceScore = StatisticsHelper.CalculateExplainedVarianceScore(actual, predicted); - LearningCurve = StatisticsHelper.CalculateLearningCurve(actual, predicted, learningCurveSteps); - PearsonCorrelation = StatisticsHelper.CalculatePearsonCorrelationCoefficient(actual, predicted); - SpearmanCorrelation = StatisticsHelper.CalculateSpearmanRankCorrelationCoefficient(actual, predicted); - KendallTau = StatisticsHelper.CalculateKendallTau(actual, predicted); - DynamicTimeWarping = StatisticsHelper.CalculateDynamicTimeWarping(actual, predicted); - - PredictionInterval = StatisticsHelper.CalculatePredictionIntervals(actual, predicted, confidenceLevel); - PredictionIntervalCoverage = StatisticsHelper.CalculatePredictionIntervalCoverage(actual, predicted, PredictionInterval.Lower, PredictionInterval.Upper); - ConfidenceInterval = StatisticsHelper.CalculateConfidenceIntervals(predicted, confidenceLevel, BestDistributionFit.DistributionType); - CredibleInterval = StatisticsHelper.CalculateCredibleIntervals(predicted, confidenceLevel, BestDistributionFit.DistributionType); - ToleranceInterval = StatisticsHelper.CalculateToleranceInterval(actual, predicted, confidenceLevel); - ForecastInterval = StatisticsHelper.CalculateForecastInterval(actual, predicted, confidenceLevel); - QuantileIntervals = StatisticsHelper.CalculateQuantileIntervals(actual, predicted, new T[] { _numOps.FromDouble(0.25), _numOps.FromDouble(0.5), _numOps.FromDouble(0.75) }); - BootstrapInterval = StatisticsHelper.CalculateBootstrapInterval(actual, predicted, confidenceLevel); - SimultaneousPredictionInterval = StatisticsHelper.CalculateSimultaneousPredictionInterval(actual, predicted, confidenceLevel); - JackknifeInterval = StatisticsHelper.CalculateJackknifeInterval(actual, predicted); - PercentileInterval = StatisticsHelper.CalculatePercentileInterval(predicted, confidenceLevel); - - Accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); - (Precision, Recall, F1Score) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + // All intermediate values computed into locals; properties are set + // together at the bottom. Before this rewrite the method read + // AdjustedR2's dependency via R2's property getter, which calls + // EnsureFullStatsComputed and re-enters this method — unbounded + // recursion + StackOverflowException in the test host. Same class + // of bug as BasicStats / ErrorStats; see those files for context. + var bestDist = StatisticsHelper.DetermineBestFitDistribution(predicted); + + T meanPredErr = StatisticsHelper.CalculateMeanPredictionError(actual, predicted); + T medianPredErr = StatisticsHelper.CalculateMedianPredictionError(actual, predicted); + + T r2 = StatisticsHelper.CalculateR2(actual, predicted); + T adjR2 = StatisticsHelper.CalculateAdjustedR2(r2, actual.Length, numberOfParameters); + T evs = StatisticsHelper.CalculateExplainedVarianceScore(actual, predicted); + var lc = StatisticsHelper.CalculateLearningCurve(actual, predicted, learningCurveSteps); + T pearson = StatisticsHelper.CalculatePearsonCorrelationCoefficient(actual, predicted); + T spearman = StatisticsHelper.CalculateSpearmanRankCorrelationCoefficient(actual, predicted); + T kendall = StatisticsHelper.CalculateKendallTau(actual, predicted); + T dtw = StatisticsHelper.CalculateDynamicTimeWarping(actual, predicted); + + (T Lower, T Upper) predInterval = StatisticsHelper.CalculatePredictionIntervals(actual, predicted, confidenceLevel); + T predIntervalCov = StatisticsHelper.CalculatePredictionIntervalCoverage(actual, predicted, predInterval.Lower, predInterval.Upper); + var confInterval = StatisticsHelper.CalculateConfidenceIntervals(predicted, confidenceLevel, bestDist.DistributionType); + var credInterval = StatisticsHelper.CalculateCredibleIntervals(predicted, confidenceLevel, bestDist.DistributionType); + var tolInterval = StatisticsHelper.CalculateToleranceInterval(actual, predicted, confidenceLevel); + var forecastInterval = StatisticsHelper.CalculateForecastInterval(actual, predicted, confidenceLevel); + var quantiles = StatisticsHelper.CalculateQuantileIntervals(actual, predicted, new T[] { _numOps.FromDouble(0.25), _numOps.FromDouble(0.5), _numOps.FromDouble(0.75) }); + var bootstrap = StatisticsHelper.CalculateBootstrapInterval(actual, predicted, confidenceLevel); + var simulPred = StatisticsHelper.CalculateSimultaneousPredictionInterval(actual, predicted, confidenceLevel); + var jackknife = StatisticsHelper.CalculateJackknifeInterval(actual, predicted); + var percentile = StatisticsHelper.CalculatePercentileInterval(predicted, confidenceLevel); + + T accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); + var (precision, recall, f1) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + + BestDistributionFit = bestDist; + MeanPredictionError = meanPredErr; + MedianPredictionError = medianPredErr; + R2 = r2; + AdjustedR2 = adjR2; + ExplainedVarianceScore = evs; + LearningCurve = lc; + PearsonCorrelation = pearson; + SpearmanCorrelation = spearman; + KendallTau = kendall; + DynamicTimeWarping = dtw; + PredictionInterval = predInterval; + PredictionIntervalCoverage = predIntervalCov; + ConfidenceInterval = confInterval; + CredibleInterval = credInterval; + ToleranceInterval = tolInterval; + ForecastInterval = forecastInterval; + QuantileIntervals = quantiles; + BootstrapInterval = bootstrap; + SimultaneousPredictionInterval = simulPred; + JackknifeInterval = jackknife; + PercentileInterval = percentile; + Accuracy = accuracy; + Precision = precision; + Recall = recall; + F1Score = f1; } /// From 0f1bb6f2afe4ceb9afd888cf0a0c1366d6b44273 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 16:40:55 -0400 Subject: [PATCH 11/29] fix(networks): resnet/vgg train adds batch dim for 3d input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ResNet/VGG's Forward() explicitly accepts 3D [C,H,W] input and expands it to 4D [1,C,H,W] before running the layer stack. Their Train() overrides, however, called TrainWithTape directly — which delegates to NeuralNetworkBase.ForwardForTraining, which does NOT add a batch dim and just runs the raw tensor through every layer. For a 3D input [3, 32, 32], the conv/pool chain preserves the rank-3 shape and the classifier's AdaptiveAveragePool + Flatten ends up producing [512, 1] (the 512 final-block channel count gets treated as a batch dim by FlattenLayer.Forward's "preserve first dim" rule). The final DenseLayer with inputSize=512 sees actualInputSize=1 via input.Shape[^1], calls EnsureWeightShapeForInput(1) which resizes weights to [1, 10], and produces [512, 10] — which then fails the loss shape check in EnsureTargetMatchesPredicted because the target is [10]. Fix: mirror Forward()'s expansion in Train() — when input is 3D, add a leading batch dim to BOTH input and target before dispatching to TrainWithTape. Any 4D input is passed through untouched. The target expansion is guarded so a caller that already provided a batched target is not double-expanded. Verified locally, all 4 of the previously-failing tests now pass: - ResNetNetwork_Train_CompletesWithoutError - ResNetNetwork_Train_LossDecreases - VGGNetwork_Train_CompletesWithoutError - VGGNetwork_Train_LossDecreases Closes the 08a NN-Classic (ResNet/VGG/DenseNet) CI shard failure from the PR #1154 triage. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/NeuralNetworks/ResNetNetwork.cs | 19 ++++++++++++++++++- src/NeuralNetworks/VGGNetwork.cs | 13 ++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/NeuralNetworks/ResNetNetwork.cs b/src/NeuralNetworks/ResNetNetwork.cs index fc27561c76..d5d8915978 100644 --- a/src/NeuralNetworks/ResNetNetwork.cs +++ b/src/NeuralNetworks/ResNetNetwork.cs @@ -521,12 +521,29 @@ public override Tensor Predict(Tensor input) /// /// The input tensor for training. /// The expected output tensor (one-hot encoded class labels). + /// + /// Accepts both 3D [C, H, W] (single unbatched example) and 4D + /// [B, C, H, W] inputs — matches 's contract. + /// When a 3D input arrives, a leading batch dimension is added so every + /// downstream layer sees the conv-standard 4D tensor. The corresponding + /// target is also expanded to [1, numClasses] so the loss sees + /// matching batch dims on both operands. Previously + /// fed the raw 3D + /// tensor straight to the layer stack, which caused the FlattenLayer to + /// treat the 512-channel dimension as a batch and produce a + /// [512, 10] prediction — failing the loss shape check in + /// EnsureTargetMatchesPredicted. + /// public override void Train(Tensor input, Tensor expectedOutput) { SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; + Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 + ? AddBatchDimension(expectedOutput) + : expectedOutput; + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { diff --git a/src/NeuralNetworks/VGGNetwork.cs b/src/NeuralNetworks/VGGNetwork.cs index 74d41f06a3..5ddacfd1b0 100644 --- a/src/NeuralNetworks/VGGNetwork.cs +++ b/src/NeuralNetworks/VGGNetwork.cs @@ -388,7 +388,18 @@ public override void Train(Tensor input, Tensor expectedOutput) SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + // Match Forward's behavior: accept 3D [C,H,W] single example and + // expand to 4D [1,C,H,W] so the layer stack (conv / pool / flatten / + // dense) receives the batched shape it expects. Without the + // expansion the FlattenLayer in the classifier head treats the + // channel dimension as a batch and produces an output with the + // wrong leading dim — fails the loss-shape check on every training + // call with a single 3D input. + Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; + Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 + ? AddBatchDimension(expectedOutput) + : expectedOutput; + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { From 2c5b83e2eb3017c2d7e21fec6ce8d769acd4598c Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 17:14:27 -0400 Subject: [PATCH 12/29] fix(networks): mobilenetv2 handles 3d input in forward/train/namedactivations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same structural bug as ResNet/VGG: MobileNetV2's Forward / Train / GetNamedLayerActivations all iterated the layer stack with the raw input. For 3D [C, H, W] inputs, BatchNormalizationLayer's channel scale (shape [1, C, 1, 1]) cannot broadcast against the 3D layout because dim 1 of the input (spatial H) doesn't match the BN's C channel count: "Tensors with shapes [16, 32, 32] and [1, 16, 1] cannot be broadcast: dimension 1 has sizes 32 and 16 (must be equal or one must be 1)." Fix: add a leading batch dimension when the caller passes a 3D input so every BN in every InvertedResidualBlock sees the 4D layout it requires, and squeeze it back off at the end of Forward so the output shape matches the caller's 3D contract. Train() expands both input and target the same way so ForwardForTraining (which iterates layers without adding batch dim) also sees the correct shape. GetNamedLayerActivations is overridden with the same expansion so the layer-by-layer probe used by NamedLayerActivations_ShouldBeNonEmpty doesn't hit the same BN broadcast error. Also fixes the test: the parameterless MobileNetV2Network constructor defaults to 1000 ImageNet classes and 224x224 input; the test probed with 3x64x64 and 10-class OutputShape. Swap in the architecture-aware overload so the classifier head matches the expected output dim. Goes from 0/17 passing on the previous config to 14/17 passing — the three remaining failures are a deeper shape-collapse issue inside the InvertedResidualBlock chain for the NamedLayerActivations probe and a perf timeout on the training tests, both of which are separate from this broadcast-shape root cause. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/NeuralNetworks/MobileNetV2Network.cs | 61 ++++++++++++++++++- .../NeuralNetworks/MobileNetV2NetworkTests.cs | 15 ++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/NeuralNetworks/MobileNetV2Network.cs b/src/NeuralNetworks/MobileNetV2Network.cs index 8a210fb3b8..e77881b0be 100644 --- a/src/NeuralNetworks/MobileNetV2Network.cs +++ b/src/NeuralNetworks/MobileNetV2Network.cs @@ -239,15 +239,64 @@ public Tensor Forward(Tensor input) if (TryForwardGpuOptimized(input, out var gpuResult)) return gpuResult; - + // Accept 3D [C, H, W] (single unbatched sample) or 4D [B, C, H, W]. + // Internal layers — especially BatchNormalizationLayer — + // broadcast their channel-scale/shift tensors as [1, C, 1, 1] and + // require the input to be 4D for that broadcast to apply along the + // channel dimension. A 3D input would otherwise line up the 16-channel + // scale tensor against the spatial H=32 dimension and throw + // "Tensors with shapes [16, 32, 32] and [1, 16, 1] cannot be + // broadcast: dimension 1 has sizes 32 and 16 (must be equal or + // one must be 1)." + // inside BatchNormalizationLayer.ApplyInferenceAnyRank. The fix + // mirrors ResNet/VGG's input-shape contract. + bool addedBatch = false; Tensor output = input; + if (input.Rank == 3) + { + output = AddBatchDimension(input); + addedBatch = true; + } + foreach (var layer in Layers) { output = layer.Forward(output); } + + // If the caller passed a 3D input we expanded to 4D; squeeze the + // leading batch dim off so the output shape matches the input's + // un-batched contract. + if (addedBatch && output.Rank >= 1 && output.Shape[0] == 1) + { + int[] squeezed = new int[output.Rank - 1]; + for (int i = 0; i < squeezed.Length; i++) squeezed[i] = output.Shape[i + 1]; + output = Engine.Reshape(output, squeezed); + } return output; } + private static Tensor AddBatchDimension(Tensor input) + { + int[] shape = input._shape; + int[] expanded = new int[shape.Length + 1]; + expanded[0] = 1; + for (int i = 0; i < shape.Length; i++) expanded[i + 1] = shape[i]; + return input.Reshape(expanded); + } + + /// + public override Dictionary> GetNamedLayerActivations(Tensor input) + { + // Mirror Forward's 3D → 4D expansion so the channel-broadcasted + // BatchNormalizationLayer inside every InvertedResidualBlock sees + // the 4D tensor it requires. Without this, callers that probe + // activations layer-by-layer with a 3D test input hit the same + // "[16,32,32] vs [1,16,1] cannot be broadcast" failure that the + // Forward override already handles. + Tensor probeInput = input.Rank == 3 ? AddBatchDimension(input) : input; + return base.GetNamedLayerActivations(probeInput); + } + /// public override Tensor Predict(Tensor input) { @@ -257,7 +306,15 @@ public override Tensor Predict(Tensor input) /// public override void Train(Tensor input, Tensor expectedOutput) { - TrainWithTape(input, expectedOutput, _optimizer); + // Mirror Forward's shape contract — expand 3D inputs (and their + // targets) to 4D so ForwardForTraining's raw layer iteration + // receives the 4D tensor BatchNormalizationLayer needs. See + // ResNet/VGG Train() for the same pattern. + Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; + Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 + ? AddBatchDimension(expectedOutput) + : expectedOutput; + TrainWithTape(processedInput, processedTarget, _optimizer); } /// diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs index 1ce24c65a8..af90d6a74f 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs @@ -1,3 +1,5 @@ +using AiDotNet.Configuration; +using AiDotNet.Enums; using AiDotNet.Interfaces; using AiDotNet.NeuralNetworks; using AiDotNet.Tests.ModelFamilyTests.Base; @@ -9,6 +11,17 @@ public class MobileNetV2NetworkTests : NeuralNetworkModelTestBase protected override int[] InputShape => [3, 64, 64]; protected override int[] OutputShape => [10]; + // The parameterless MobileNetV2Network() constructor defaults to 1000 + // ImageNet classes and 224x224 input — incompatible with this test's + // small 3x64x64 probe and 10-class OutputShape assertion. Use the + // architecture-aware overload so the network classifier head matches + // the test's expected output dimension. protected override INeuralNetworkModel CreateNetwork() - => new MobileNetV2Network(); + => new MobileNetV2Network( + new NeuralNetworkArchitecture( + inputType: InputType.ThreeDimensional, + taskType: NeuralNetworkTaskType.MultiClassClassification, + inputHeight: 64, inputWidth: 64, inputDepth: 3, + outputSize: 10), + MobileNetV2Configuration.CreateStandard(numClasses: 10)); } From 46f2f2f925eff92aed26f0f49c527e82864cfb18 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 17:25:59 -0400 Subject: [PATCH 13/29] test(networks): instructorembedding test shape matches 768-dim model InstructorEmbedding's default ctor builds a 768-dim transformer (inputSize=768, outputSize=768) but the test inherited the base class's default InputShape=[1, 4] and OutputShape=[1, 1]. The training tests fed a [1, 4] input to a 768-dim model and a [1, 1] target that the loss function then tried to subtract from the model's [1, 768] prediction, throwing "Tensor shapes must match. Got [1, 768] and [1, 1]." in MeanSquaredErrorLoss.ComputeTapeLoss. Fix: override InputShape/OutputShape to the model's actual 768-dim embedding layout so input, prediction, and target all align. Closes the InstructorEmbedding part of the "ModelFamily - NeuralNetworks" CI shard failure from the PR #1154 triage (remaining failures in that shard are MobileNetV2 and are addressed in the previous commit). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../NeuralNetworks/InstructorEmbeddingTests.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs index 41c71e3963..4b6b6d227f 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs @@ -6,6 +6,16 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class InstructorEmbeddingTests : EmbeddingModelTestBase { + // InstructorEmbedding's default ctor wires a 768-dim transformer + // (inputSize=768, outputSize=768). The test base defaults to [1, 4] + // input and [1, 1] target, which caused the loss computation to try + // subtracting a [1, 768] prediction from a [1, 1] target and throw + // "Tensor shapes must match. Got [1, 768] and [1, 1]." in + // MeanSquaredErrorLoss.ComputeTapeLoss. Align the test shapes with + // the model's actual input/output dimensions. + protected override int[] InputShape => [1, 768]; + protected override int[] OutputShape => [1, 768]; + protected override INeuralNetworkModel CreateNetwork() => new InstructorEmbedding(); } From 08b86e7247891abe9cdca446cea1313882f718a6 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 18:25:19 -0400 Subject: [PATCH 14/29] fix(networks): convolutionalneuralnetwork train adds batch dim for 3d input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same 3D-input bug as ResNet/VGG/MobileNetV2: CNN's Train() called TrainWithTape with the raw 3D [C, H, W] tensor. ForwardForTraining iterates layers without a shape-adjustment step, so the final FlattenLayer treats the 32-channel dimension as a batch (preserve-first-dim rule) and produces a [32, 10] prediction against a [10] one-hot target — fails EnsureTargetMatchesPredicted with "Target shape dimension 0 (10) does not match predicted shape dimension 0 (32)." Fix: expand 3D input to 4D before dispatching to TrainWithTape, and expand the target too when the caller provided it without a batch dim. All 5 previously-failing CNN tests pass locally: - TrainingError_ShouldNotExceedTestError - Training_ShouldReduceLoss - Training_ShouldChangeParameters - GradientFlow_ShouldBeNonZeroAndFinite - ForwardPass_ShouldBeFinite_AfterTraining Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ConvolutionalNeuralNetwork.cs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs index da125b4bae..d161f965bf 100644 --- a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs +++ b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs @@ -260,7 +260,21 @@ public override void Train(Tensor input, Tensor expectedOutput) SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + // Accept both 3D [C, H, W] and 4D [B, C, H, W] inputs — matches the + // Forward contract. ForwardForTraining runs the raw tensor through + // every layer without adjusting rank, so a 3D input would make the + // final FlattenLayer treat the 32-channel dimension as a batch + // (flatten preserves dim 0), producing a [32, 10] prediction + // against a [10] one-hot target — fails the loss shape check: + // "Target shape dimension 0 (10) does not match predicted + // shape dimension 0 (32)." + // Mirror ResNet/VGG/MobileNetV2's fix: expand input and target to + // 4D (and 2D respectively) before dispatching to TrainWithTape. + Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; + Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 + ? AddBatchDimension(expectedOutput) + : expectedOutput; + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { @@ -268,6 +282,15 @@ public override void Train(Tensor input, Tensor expectedOutput) } } + private static Tensor AddBatchDimension(Tensor t) + { + int[] shape = t._shape; + int[] expanded = new int[shape.Length + 1]; + expanded[0] = 1; + for (int i = 0; i < shape.Length; i++) expanded[i + 1] = shape[i]; + return t.Reshape(expanded); + } + /// /// Updates the parameters of the network based on the calculated gradients. /// From 15dc66cd9aaf1e56a9a7bf1712b8ae6f2b217beb Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 19:05:02 -0400 Subject: [PATCH 15/29] fix(networks): unet3d decoder channel count + test output shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related problems surfaced by every UNet3D test: 1. LayerHelper.CreateDefaultUNet3DLayers — the decoder path declared the first Conv3D of each non-bottleneck-adjacent block with `inChannels = encoderFilters[block + 1] * 2`. The "*2" was there to account for a full U-Net concatenating skip connections from the encoder at each decoder level. This implementation does NOT actually perform the concatenation, so the preceding decoder block's Second-Conv3D emitted encoderFilters[block + 1] channels, not double that. Every CI call (and every local Predict) hit "Input channels (128) must match kernel in_channels (256)" in the first decoder block after the one adjacent to the bottleneck. Fix: drop the "*2" so the declared in_channels match the tensors that actually flow through. Concatenating real skip connections is a separate architectural improvement. 2. UNet3DTests — OutputShape declared as [1], treating the network as a classifier, but UNet3D is a per-voxel segmentation model whose final 1x1x1 Conv3D emits [numClasses, D, H, W] per sample. With default numClasses=1 and 32³ voxel grid, every training test tried to subtract a [1, 32, 32, 32] prediction from a [1] target and threw "Tensor shapes must match. Got [1, 32, 32, 32] and [1]." Fix: OutputShape → [1, 32, 32, 32] so input, prediction, and target all line up. Goes from 0/17 passing on UNet3D to 12/17. The five remaining failures are separate issues (NaN during training for this conv stack, metadata parity) that are independent of these two root causes. Closes 7 of the 8 UNet3D failures from the PR #1154 CI triage that were all attributed to the "Input channels (128) vs (256)" error. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Helpers/LayerHelper.cs | 16 ++++++++++++++-- .../NeuralNetworks/UNet3DTests.cs | 12 ++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/Helpers/LayerHelper.cs b/src/Helpers/LayerHelper.cs index b3b537a83c..1563132dc4 100644 --- a/src/Helpers/LayerHelper.cs +++ b/src/Helpers/LayerHelper.cs @@ -4207,11 +4207,23 @@ public static IEnumerable> CreateDefaultUNet3DLayers( // ============== DECODER PATH ============== // Each decoder block: Upsample3D -> Conv3D -> Conv3D - // Note: Skip connections need to be handled by the network model + // + // Note: a full U-Net would concatenate encoder skip-connections at + // each decoder level, doubling the channel count into the first + // Conv3D. This implementation does NOT actually perform the + // concatenation, so the "*2" previously applied to + // encoderFilters[block + 1] told the Upsample and first-Conv3D to + // expect twice the channels they would actually receive from the + // preceding decoder block's Second-Conv3D output. That produced + // "Input channels (128) must match kernel in_channels (256)" + // at the first decoder block after the bottleneck-adjacent one. + // Matching the actual tensor-channel count (= encoderFilters[block + 1]) + // keeps the stack consistent; adding real skip concatenation is a + // separate architectural improvement tracked independently. for (int block = numEncoderBlocks - 2; block >= 0; block--) { int outputFilters = encoderFilters[block]; - int inChannels = block == numEncoderBlocks - 2 ? bottleneckFilters : encoderFilters[block + 1] * 2; + int inChannels = block == numEncoderBlocks - 2 ? bottleneckFilters : encoderFilters[block + 1]; // Upsample3D to increase resolution yield return new Upsample3DLayer( diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs index 79b80a3119..107bdd4ef1 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs @@ -6,9 +6,17 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class UNet3DTests : NeuralNetworkModelTestBase { - // UNet3D default: 32x32x32 voxels, 1 channel, outputSize=1 + // UNet3D is a per-voxel segmentation network: it emits one class + // prediction per input voxel, so the output carries the same spatial + // dimensions as the input. The final 1x1x1 Conv3D produces + // [numClasses, D, H, W] per sample — for the default single-class + // config that is [1, 32, 32, 32], NOT [1] (which is what the previous + // OutputShape claim produced). Without this correction the training + // tests threw "Tensor shapes must match. Got [1, 32, 32, 32] and [1]" + // when the loss tried to subtract a per-voxel prediction from a + // scalar target. protected override int[] InputShape => [1, 32, 32, 32]; - protected override int[] OutputShape => [1]; + protected override int[] OutputShape => [1, 32, 32, 32]; protected override INeuralNetworkModel CreateNetwork() => new UNet3D(); From b046eb257b00109f3f3d3309b0f857bcc4b49773 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 19:13:12 -0400 Subject: [PATCH 16/29] fix(gp): escalating cholesky jitter for sparsegaussianprocess.fit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ky = Kuu + D·Kuf·Kuf^T is only positive-semi-definite in exact arithmetic, so floating-point roundoff on the combined matrix routinely pushes the smallest eigenvalue just below zero and CholeskyDecomposition throws "Matrix is not positive definite" on every SparseGaussianProcess fit. Kuu already gets a constant 1e-4 jitter before its Cholesky, but the Ky path had none — that produced the six SparseGaussianProcessTests failures in the PR #1156 CI shard. Add a PyTorch/GPyTorch-style escalating jitter schedule (1e-6 → 1e-4 → 1e-2 → 1e-1, scaled by the matrix trace so it's invariant to kernel amplitude) and retry the Cholesky after each increment. Geometric escalation instead of a single larger constant keeps the numerical error introduced for already-well-conditioned matrices minimal while still rescuing the borderline cases. Goes from 7/16 passing to 14/16 on SparseGaussianProcessTests. Remaining two failures are separate bugs (predictive mean is NaN, not a PD-matrix issue) tracked independently. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../SparseGaussianProcess.cs | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/GaussianProcesses/SparseGaussianProcess.cs b/src/GaussianProcesses/SparseGaussianProcess.cs index 520370d485..bcba9ec93a 100644 --- a/src/GaussianProcesses/SparseGaussianProcess.cs +++ b/src/GaussianProcesses/SparseGaussianProcess.cs @@ -204,7 +204,44 @@ public override void Fit(Matrix X, Vector y) Ky[j, i] = avg; } - var choleskyKy = new CholeskyDecomposition(Ky); + // Add diagonal jitter before Cholesky for the same reason Kuu gets + // it above: Ky = Kuu + D·Kuf·Kuf^T is only positive-semi-definite + // in exact arithmetic (when rank(D·Kuf·Kuf^T) < m, which is typical + // when the number of inducing points matches the data dimensionality). + // Floating-point roundoff then pushes the smallest eigenvalue just + // below zero and CholeskyDecomposition throws "Matrix is not + // positive definite" — this was the root cause of all six + // SparseGaussianProcessTests failures in the PR #1156 CI shard. + // Escalate jitter on failure (PyTorch / GPyTorch pattern): start at + // 1e-6·trace, retry at 1e-4·trace, then 1e-2·trace before giving up. + T traceScale = _numOps.Zero; + for (int i = 0; i < Ky.Rows; i++) + traceScale = _numOps.Add(traceScale, Ky[i, i]); + traceScale = _numOps.Divide(traceScale, _numOps.FromDouble(Ky.Rows)); + + CholeskyDecomposition? choleskyKy = null; + double[] jitterSchedule = { 1e-6, 1e-4, 1e-2, 1e-1 }; + foreach (var scale in jitterSchedule) + { + T jitterAmt = _numOps.Multiply(traceScale, _numOps.FromDouble(scale)); + for (int i = 0; i < Ky.Rows; i++) + Ky[i, i] = _numOps.Add(Ky[i, i], jitterAmt); + try + { + choleskyKy = new CholeskyDecomposition(Ky); + break; + } + catch (ArgumentException) + { + // Escalate by adding more jitter on top of what we already + // added; the next iteration's jitterAmt is added to the + // already-augmented diagonal, growing geometrically. + continue; + } + } + if (choleskyKy is null) + throw new ArgumentException( + "Matrix is not positive definite — Ky remained non-PD even after jittering at scales up to 0.1 × trace."); var alpha = choleskyKy.Solve(DKuf.Multiply(y)); // Store necessary components for prediction From 290090dab9fb01412339e7336edb6c095994cb1e Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 19:20:38 -0400 Subject: [PATCH 17/29] fix(generators): correct audio/video modeldomain ordinal in testscaffoldgenerator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ModelDomain enum order is General=0, Vision=1, Language=2, Audio=3, Video=4, Multimodal=5. The scaffold generator had Audio and Video ordinals swapped in three places: 1. Line 1495 — treats Domain=3 as "temporal video" and emits `throw new NotImplementedException(...)` in the test's CreateNetwork. Audio is 3, not 4, so EVERY audio model (PlayHT, Bark, StableAudio, etc.) got a NotImplementedException factory instead of a working architecture. Ten PlayHTTests failures on PR #1156 traced back to this single line. 2. Line 1520 — `isAudio = Domains.Contains(4)`. Should be 3. 3. Line 1633 — `isVideoModel = Domains.Contains(3)`. Should be 4. All three sites now use the correct ordinals (Audio=3, Video=4). This aligns the generator with the enum and the facade/customization pattern the project prefers over hard-coded factories — every audio model's test can now construct a real Architecture and run the test body (which exposes the real model-specific failures downstream, where they can be fixed in the model code rather than hidden behind a runtime factory stub). PlayHTTests go from 0/21 passing (all NotImplementedException) to 2/21 (metadata/parameter-count tests now execute). The remaining 19 failures are a separate PlayHT LayerNorm shape-mismatch issue that can be addressed independently now that the tests actually run. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/AiDotNet.Generators/TestScaffoldGenerator.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/AiDotNet.Generators/TestScaffoldGenerator.cs b/src/AiDotNet.Generators/TestScaffoldGenerator.cs index 6b0194a23a..4d34e7e7f6 100644 --- a/src/AiDotNet.Generators/TestScaffoldGenerator.cs +++ b/src/AiDotNet.Generators/TestScaffoldGenerator.cs @@ -1492,7 +1492,7 @@ private static void EmitGeneratedTestClass( } } } - else if (model.Domains.Contains(3) && !model.Tasks.Contains(35)) + else if (model.Domains.Contains(4) && !model.Tasks.Contains(35)) { // Temporal video models (ActionRecognition=22, VideoGeneration=41, etc.) // need a 4D [frames, channels, height, width] input shape, but @@ -1500,6 +1500,13 @@ private static void EmitGeneratedTestClass( // Rather than silently emit a mismatched 3D architecture alongside a // 4D InputShape, route these to a runtime placeholder until the // architecture type can represent a temporal dimension. + // + // The enum ordinal for ModelDomain.Video is 4 + // (General=0, Vision=1, Language=2, Audio=3, Video=4, ...). + // This check previously used 3, which incorrectly flagged every + // *audio* model (PlayHT, Bark, etc.) as "temporal video" and + // emitted a NotImplementedException factory — ten PlayHTTests + // failures on PR #1156 traced to this off-by-one. constructorExpr = "throw new System.NotImplementedException(" + $"\"'{GeneratorHelpers.StripGenericSuffix(model.ClassName)}' is a temporal video model; NeuralNetworkArchitecture cannot express its 4D [frames, channels, height, width] input. Implement this factory manually.\")"; } @@ -1510,7 +1517,7 @@ private static void EmitGeneratedTestClass( // others default to OneDimensional. Temporal video is handled above. needsArchitectureUsing = true; bool isVision = model.Domains.Contains(1) || model.Domains.Contains(11); // Vision=1, ThreeD=11 - bool isAudio = model.Domains.Contains(4); // Audio=4 + bool isAudio = model.Domains.Contains(3); // Audio=3 (enum ordinal, not Video=4) bool isFrameInterp = model.Tasks.Contains(35); // FrameInterpolation → 3D input string inputTypeExpr; @@ -1623,11 +1630,12 @@ private static void EmitGeneratedTestClass( // Override InputShape/OutputShape for domain-appropriate test data. // Vision/Video/3D models need [C, H, W]; default is [1, 4]. - bool isVideoModel = model.Domains.Contains(3); + // Enum ordinals: General=0, Vision=1, Language=2, Audio=3, Video=4. + bool isVideoModel = model.Domains.Contains(4); // Video=4 (was incorrectly 3) bool isFrameInterpModel = model.Tasks.Contains(35); // FrameInterpolation bool isTemporalVideoModel = isVideoModel && !isFrameInterpModel; bool isVisionModel = model.Domains.Contains(1) || model.Domains.Contains(11); - bool isAudioModel = model.Domains.Contains(4); + bool isAudioModel = model.Domains.Contains(3); // Audio=3 (was incorrectly 4) if (isTemporalVideoModel) { // Temporal video: [frames, channels, height, width] From 42b48ad3725592dc0e59675f6cc07cc22bb3cce0 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 19:29:34 -0400 Subject: [PATCH 18/29] test(neuralnetworks): align word2vec test shapes with softmax vocab head word2vec's default constructor uses vocabsize=10000. the final layer emits a 10000-dim softmax over the vocabulary, so per-sample output is [1, 10000], not the [1, 1] implied by the base-class default. align input/output shape so outputdimension_shouldmatchexpectedshape compares the right tensors. --- .../ModelFamilyTests/NeuralNetworks/Word2VecTests.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs index 8ba858b420..87bdddaee9 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs @@ -6,6 +6,14 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class Word2VecTests : NeuralNetworkModelTestBase { + // Word2Vec's default ctor uses vocabSize=10000 — the last layer emits + // a 10000-dim softmax over the vocabulary, so predicted output length + // is 10000 per sample, not the [1, 1] implied by the base-class + // default OutputShape. Align both sides so + // OutputDimension_ShouldMatchExpectedShape compares like with like. + protected override int[] InputShape => [1, 4]; + protected override int[] OutputShape => [1, 10000]; + protected override INeuralNetworkModel CreateNetwork() => new Word2Vec(); } From ca646c8aa016f605d7cfde94153c3a7b810ea3cf Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 19:42:27 -0400 Subject: [PATCH 19/29] test(ner): emit 768-dim scaffolded shapes for transformer ner models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transformernerbase, spanbasedernbase, and the lstm-crf family all validate token embeddings against their options.hiddendimension (768 by default, 100 for lstm-crf). the auto-scaffolded test base inherited [1, 4] as inputshape, so multiheadattention threw "input embedding dimension (4) does not match weight dimension (768)" before any downstream logic could run — the reported scibertner training-error regression on pr #1156. emit inputshape = [8, 768] for transformerner/spanbasedner and [8, 100] for sequencelabelingner in the test scaffolder. add a manual tinybertnertests with [8, 312] so the one model that overrides hiddendimension still gets covered. --- .../TestScaffoldGenerator.cs | 17 ++++++++++++ .../NeuralNetworks/TinyBERTNERTests.cs | 26 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs diff --git a/src/AiDotNet.Generators/TestScaffoldGenerator.cs b/src/AiDotNet.Generators/TestScaffoldGenerator.cs index 4d34e7e7f6..4ffd29ee50 100644 --- a/src/AiDotNet.Generators/TestScaffoldGenerator.cs +++ b/src/AiDotNet.Generators/TestScaffoldGenerator.cs @@ -1668,6 +1668,23 @@ private static void EmitGeneratedTestClass( sb.AppendLine($" protected override int[] InputShape => new[] {{ {dim} }};"); sb.AppendLine(" protected override int[] OutputShape => new[] { 4 };"); } + else if (family == TestFamily.TransformerNER || family == TestFamily.SpanBasedNER) + { + // TransformerNERBase and SpanBasedNERBase both default to + // HiddenDimension=768 (BERT-base). Inputs are validated as + // [seqLen, 768], so the base-class default [1, 4] causes a + // hard "embedding dim mismatch" failure inside MultiHeadAttention + // before any downstream logic runs. Use a short sequence to + // keep the test fast while matching the model's expected + // embedding size. Models with non-default hidden dimensions + // (TinyBERT=312, etc.) need a manual test override. + sb.AppendLine(" protected override int[] InputShape => new[] { 8, 768 };"); + } + else if (family == TestFamily.SequenceLabelingNER) + { + // LSTM-CRF family defaults to EmbeddingDimension=100. + sb.AppendLine(" protected override int[] InputShape => new[] { 8, 100 };"); + } sb.AppendLine($" protected override {returnTypeCode} {factoryMethodName}()"); sb.AppendLine(factoryBody); diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs new file mode 100644 index 0000000000..a105d4da4c --- /dev/null +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs @@ -0,0 +1,26 @@ +using AiDotNet.Enums; +using AiDotNet.Interfaces; +using AiDotNet.NER.TransformerBased; +using AiDotNet.NeuralNetworks; +using AiDotNet.Tests.ModelFamilyTests.Base; + +namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; + +public class TinyBERTNERTests : TransformerNERTestBase +{ + // TinyBERT overrides TransformerNEROptions.HiddenDimension to 312 + // (vs the 768 default used by BERT-base). The generator's default + // [8, 768] InputShape would fail MultiHeadAttention weight matching, + // so we pin it to 312 here and construct the model via its + // architecture-only constructor (which wires up CreateTinyBERTDefaults + // with the 312-dim attention weights). + protected override int[] InputShape => [8, 312]; + + protected override INeuralNetworkModel CreateNetwork() + => new TinyBERTNER( + new NeuralNetworkArchitecture( + inputType: InputType.OneDimensional, + taskType: NeuralNetworkTaskType.Regression, + inputSize: 128, + outputSize: 4)); +} From 6173705254e1059fe5a92cbc7797f39dc2fc1d90 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 20:05:27 -0400 Subject: [PATCH 20/29] fix(layers): default rnn head should use identityactivation, not relu-via-null recurrent network's default layer stack terminated in a dense layer constructed with activationfunction:null, which the dense ctor substitutes with relu. the preceding two tanh recurrent layers produce small mixed-sign activations (range ~[-0.16, 0.16] on random input), and relu then clips the single-output regression head to exactly 0 for essentially any input. that is why scaledinput_shouldchangeoutput and differentinputs_shouldproducedifferentoutputs saw identical zero outputs for distinct inputs on recurrentneuralnetworktests. pass an explicit identityactivation so the dense head stays linear. the task-appropriate softmax/sigmoid activation layer emitted after it remains unchanged. --- src/Helpers/LayerHelper.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Helpers/LayerHelper.cs b/src/Helpers/LayerHelper.cs index 1563132dc4..7658b92283 100644 --- a/src/Helpers/LayerHelper.cs +++ b/src/Helpers/LayerHelper.cs @@ -2579,11 +2579,18 @@ public static IEnumerable> CreateDefaultRNNLayers(NeuralNetworkArchite // RNN layers output [seqLen, hiddenSize], but Dense layer expects [hiddenSize] yield return new SequenceLastLayer(hiddenSize); - // Add the final Dense Layer to map to output size + // Add the final Dense Layer to map to output size. + // Use an explicit IdentityActivation so the Dense output keeps its sign — + // passing activationFunction:null falls through to DenseLayer's ReLU + // default, which clipped all regression outputs at zero. With ReLU + // the 2-layer tanh stack produced small mixed-sign values that the + // final Dense then zeroed, so ScaledInput_ShouldChangeOutput saw + // output=0 for both the raw and the 10x input and reported "forward + // pass may ignore input values". yield return new DenseLayer( inputSize: hiddenSize, outputSize: outputSize, - activationFunction: null + activationFunction: new IdentityActivation() ); // Task-appropriate final activation From d964ea4880e403a3772b11234e5b7c5f1278f887 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 22:01:34 -0400 Subject: [PATCH 21/29] fix(memorynetwork): seed memory and wire training through the memory-aware flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit two root causes made every memorynetwork prediction identical regardless of input, and the training path diverge from the prediction path: 1. _memory was initialized as a zero matrix. memoryreadlayer computes keys · memory^t, so with zero memory every attention score is zero, softmax produces a uniform distribution, and attentionweights · memory reads back zero — every subsequent layer saw the same constant vector. scaledinput_shouldchangeoutput and differentinputs_ shouldproducedifferentoutputs both reported the network ignored its input. seed _memory with small xavier-scale random values so there is something non-trivial to attend over on the very first forward pass. 2. predict specialcased memoryread/memorywritelayer to pass the memory tensor and reshaped rank-1 input to [1, n], but train went through the base trainwithtape → forwardfortraining path which did neither, so training crashed ("tensormatmul requires tensors of rank >= 2") or silently read from an identity-memory fallback. factor the shared layer walk into runlayers() and override forwardfortraining so train and predict share the same memory plumbing. locally memorynetworktests goes from 9 failing → 2 (the remaining two are the known memoryreadlayer deserialization gap and namedlayeractivations, tracked separately). --- src/NeuralNetworks/MemoryNetwork.cs | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/NeuralNetworks/MemoryNetwork.cs b/src/NeuralNetworks/MemoryNetwork.cs index 111ad8e86d..fce6d568a7 100644 --- a/src/NeuralNetworks/MemoryNetwork.cs +++ b/src/NeuralNetworks/MemoryNetwork.cs @@ -187,6 +187,21 @@ public MemoryNetwork(NeuralNetworkArchitecture architecture, int memorySize, _embeddingSize = embeddingSize; _memory = new Matrix(_memorySize, _embeddingSize); + // Seed memory with small random values so MemoryReadLayer has something + // non-trivial to attend over on the first forward pass. Leaving memory + // as zeros makes keys · memory^T identically zero, so softmax produces + // a uniform attention distribution and readValues = attention · memory + // is also zero — every subsequent layer then sees the same constant + // input and the network cannot discriminate between its own inputs, + // which was the source of the scaledinput_shouldchangeoutput and + // differentinputs_shouldproducedifferentoutputs failures. Use a small + // Xavier-scale so downstream layers see a reasonable magnitude. + var memRand = AiDotNet.Tensors.Helpers.RandomHelper.CreateSecureRandom(); + double memScale = Math.Sqrt(2.0 / (_memorySize + _embeddingSize)); + for (int r = 0; r < _memorySize; r++) + for (int c = 0; c < _embeddingSize; c++) + _memory[r, c] = NumOps.FromDouble((memRand.NextDouble() * 2 - 1) * memScale); + InitializeLayers(); } @@ -308,6 +323,24 @@ public override Tensor Predict(Tensor input) // Set to inference mode SetTrainingMode(false); + return RunLayers(input); + } + + /// + public override Tensor ForwardForTraining(Tensor input) + { + // The base class version walks Layers with the plain single-arg + // Forward, which misses the Memory{Read,Write}Layer memory-tensor + // plumbing and skips the rank-1 → rank-2 reshape — both of which + // Predict does. Without this override the tape-based training path + // either crashes ("TensorMatMul requires tensors of rank >= 2") + // or silently reads from an identity-memory fallback that has + // nothing to do with _memory. + return RunLayers(input); + } + + private Tensor RunLayers(Tensor input) + { // Ensure 2D input for memory operations (TensorMatMul requires rank >= 2) if (input.Rank == 1) input = input.Reshape([1, input.Shape[0]]); From 0cf77807cc496b119b49c973cef2515d469ff544 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 22:17:57 -0400 Subject: [PATCH 22/29] fix(quantumnn): migrate training to trainwithtape and use identity on final dense MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit quantumneuralnetworktests was failing 10/17 because train called _trainoptimizer.updateparameters(layers) without first running a backward pass, tripping "backward pass must be called before updating parameters" inside each dense layer's legacy per-learning-rate update path. switch train to trainwithtape, matching resnet/vgg/mobilenetv2. the quantum default layer stack also terminated its final dense in the generator with activationfunction:null (→ relu), so regression-task output got clipped at zero before the task-specific final activation layer could run. promote that dense to identityactivation so the subsequent activationlayer owns the non-linearity, same fix pattern as the rnn regression head. locally qnn goes from 10 failing → 5 (remaining five look like a deeper input-independent forward pass — separate issue). --- src/Helpers/LayerHelper.cs | 9 ++++++-- src/NeuralNetworks/QuantumNeuralNetwork.cs | 26 +++++++++++++--------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Helpers/LayerHelper.cs b/src/Helpers/LayerHelper.cs index 7658b92283..6ab75ca5f4 100644 --- a/src/Helpers/LayerHelper.cs +++ b/src/Helpers/LayerHelper.cs @@ -2724,11 +2724,16 @@ public static IEnumerable> CreateDefaultQuantumNetworkLayers( yield return new QuantumLayer(hiddenSize, quantumDim, numQubits); yield return new MeasurementLayer(quantumDim); - // Final dense layer to map to output size + // Final dense layer to map to output size. Use identity so the + // task-appropriate activation below owns the output non-linearity + // — passing null falls through to DenseLayer's ReLU default and + // clips the pre-activation logits, which collapsed + // scaledinput_shouldchangeoutput / differentinputs for quantum + // regression models. yield return new DenseLayer( inputSize: quantumDim, outputSize: outputSize, - activationFunction: null + activationFunction: new IdentityActivation() ); // Final activation based on task type diff --git a/src/NeuralNetworks/QuantumNeuralNetwork.cs b/src/NeuralNetworks/QuantumNeuralNetwork.cs index 9866c1bbb7..838cf1d11c 100644 --- a/src/NeuralNetworks/QuantumNeuralNetwork.cs +++ b/src/NeuralNetworks/QuantumNeuralNetwork.cs @@ -281,20 +281,26 @@ public override Tensor Predict(Tensor input) /// public override void Train(Tensor input, Tensor expectedOutput) { - // Set training mode on all layers for proper gradient storage SetTrainingMode(true); foreach (var layer in Layers) layer.SetTrainingMode(true); - // Forward pass - var prediction = Predict(input); - - // Calculate and set the loss - LastLoss = CalculateLoss(prediction, expectedOutput); - - // Tape-based training handles gradient computation - var gradients = new List>(); - UpdateQuantumParameters(gradients); + try + { + // Use the tape-based training path like other networks. The previous + // imperative implementation ran Predict() and then called the + // optimizer's UpdateParameters(Layers), which dispatched to each + // layer's UpdateParameters(learningRate) — that expects a prior + // Backward() to have populated the per-layer gradient tensors, + // but none of that happens here, so every training call failed + // with "Backward pass must be called before updating parameters." + _trainOptimizer ??= new AdamOptimizer, Tensor>(this); + TrainWithTape(input, expectedOutput, _trainOptimizer); + } + finally + { + SetTrainingMode(false); + } } /// From 9c5d698289366b16199109f92c2e1abbf3ed278d Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 22:32:20 -0400 Subject: [PATCH 23/29] fix(diffusion): upscaleavideo inputconv should match latent channels, not concat width MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit upscaleavideomodel set input_channels=8 to describe the "concat latent+low-res conditioning" path from the reference paper, but forwardvideounet adds the image condition via the _imagecondprojection dense layer *after* _inputconv, not by concatenating before it. the first conv was therefore sized for 8 channels while ever actually seeing 4, and the 14 upscaleavideomodeltests cases on the diffusion a-i shard all failed with "expected input depth 8, but got 4". pin input_channels to latent_channels so the conv weight shape matches what the forward pass feeds it. this exposes a downstream film projection width mismatch tracked separately (videounetpredictor.applyfilmconditioning) — fixing that is the next step. --- .../SuperResolution/UpscaleAVideoModel.cs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs b/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs index d335168816..b0e8429596 100644 --- a/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs +++ b/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs @@ -134,13 +134,20 @@ public class UpscaleAVideoModel : VideoDiffusionModelBase private const int CROSS_ATTENTION_DIM = 1024; /// - /// Input channels for the Video U-Net (8 = 4 latent + 4 low-res conditioning). + /// Input channels for the Video U-Net input convolution (4 = latent channels). /// /// - /// The Video U-Net receives concatenated latent noise and downscaled low-resolution - /// video frames as conditioning, doubling the standard 4 latent channels. + /// The reference implementation describes this model as using 8 input channels + /// (4 latent + 4 downscaled low-res conditioning), but that design expects the + /// conditioning to be concatenated with the latents before the first conv. + /// Our VideoUNetPredictor.ForwardVideoUNet adds the image condition via + /// _imageCondProjection *after* the input conv instead — so the input conv + /// sees only the latent channels. Pinning INPUT_CHANNELS to LATENT_CHANNELS + /// keeps the conv weight shape consistent with what ForwardVideoUNet actually + /// feeds it, fixing the "Expected input depth 8, but got 4" crash in the + /// UpscaleAVideoModel tests. /// - private const int INPUT_CHANNELS = 8; + private const int INPUT_CHANNELS = LATENT_CHANNELS; /// /// Base channel count for the Video U-Net (320). From dee79b6e711c372e2de0e8ae39ceb3aaf96514a0 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 22:40:07 -0400 Subject: [PATCH 24/29] fix(diffusion): videounet spatial resblock must mix channels, not width MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createspatialresblock wrapped a lazydense(inchannels, outchannels), but denselayer projects the *last* dimension of its input. for a 4d feature map [b, c, h, w] that is the width axis, not the channel axis — so the resblock silently scrambled width into outchannels while leaving the channel count untouched. the next timecondprojection was sized for the planned outchannels, so applyfilmconditioning saw "expected 2*c, got 2*outc" and threw "film conditioning projection width mismatch: expected 640, got 1280" across upscaleavideo and streamingt2v tests. switch to a 1x1 lazyconv2d — the standard channel-mixing primitive. it consumes [b, inchannels, h, w] and produces [b, outchannels, h, w] without touching spatial dims, so downstream film projections receive a feature map with the channel count they were sized for. follow-ups (separate): multihead attention, temporal attention, and cross-attention layers still receive the 4d tensor directly without reshape, which surfaces as input-dim mismatches further down the forward pass. --- .../NoisePredictors/VideoUNetPredictor.cs | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs b/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs index 3a753b1e14..21408f1740 100644 --- a/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs +++ b/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs @@ -1019,7 +1019,28 @@ private Tensor StackFrames(List> frames) private ILayer CreateSpatialResBlock(int inChannels, int outChannels) { - return LazyDense(inChannels, outChannels, new SiLUActivation()); + // A diffusion spatial ResBlock transforms the channel dimension of a + // 4D [B, C, H, W] feature map. The previous implementation used + // LazyDense(inChannels, outChannels), but DenseLayer projects the + // *last* dimension of its input — for a 4D tensor that is the width + // axis, not channels. The block therefore left C unchanged while + // scrambling W to outChannels, and every subsequent TimeCondProjection + // (sized for the planned outChannels) saw a feature map still at the + // incoming channel count — hence "FiLM conditioning projection width + // mismatch: expected 640, got 1280" on upscaleavideomodel and + // streamingt2vmodel tests. + // A 1x1 Conv2D is the standard channel-mixing primitive: it consumes + // [B, inChannels, H, W] and produces [B, outChannels, H, W] while + // leaving spatial dims alone. + return LazyConv2D( + inputDepth: inChannels, + inputHeight: 1, + inputWidth: 1, + outputDepth: outChannels, + kernelSize: 1, + stride: 1, + padding: 0, + activation: new SiLUActivation()); } /// From e798e07ffb6e7f0b08d80d6ca6fb0626ae55e1a3 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 22:56:42 -0400 Subject: [PATCH 25/29] fix(serialization): register memoryread and memorywrite layers for deserialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clone()-style roundtrips on memorynetwork crashed with "layer type memoryreadlayer is not supported for deserialization (no known constructor found)" because deserializationhelper.createlayerfromtype had no explicit arm for either memoryread or memorywrite layer, and the default fallback tries a ctor(int[]) that neither layer exposes. add cases for both. memoryreadlayer uses a (inputdim, memorydim, outputdim, iactivation) ctor and memorywritelayer uses (inputdim, memorydim, iactivation). pick memorydim from a "memorydimension" metadata key when present, otherwise reuse the output dim — which matches how memorynetwork wires its memoryreadlayer (embeddingsize for all three dims). --- src/Helpers/DeserializationHelper.cs | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/Helpers/DeserializationHelper.cs b/src/Helpers/DeserializationHelper.cs index 4991df29d0..382245d0d7 100644 --- a/src/Helpers/DeserializationHelper.cs +++ b/src/Helpers/DeserializationHelper.cs @@ -466,6 +466,40 @@ public static ILayer CreateLayerFromType(string layerType, int[] inputShap { instance = CreateMultiLoRAAdapter(type, inputShape, outputShape, additionalParams); } + else if (genericDef == typeof(NeuralNetworks.Layers.MemoryReadLayer<>)) + { + // MemoryReadLayer(int inputDimension, int memoryDimension, int outputDimension, IActivationFunction?) + // memoryDimension is a free parameter — the default MemoryNetwork wires + // input == memory == output == embeddingSize, so fall back to output size + // if the serialized metadata doesn't pin it explicitly. + int inputDim = inputShape[0]; + int outputDim = outputShape[0]; + int memoryDim = TryGetInt(additionalParams, "MemoryDimension") ?? outputDim; + + var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType }); + if (ctor is null) + { + throw new InvalidOperationException("Cannot find MemoryReadLayer constructor with (int, int, int, IActivationFunction)."); + } + object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); + instance = ctor.Invoke(new object?[] { inputDim, memoryDim, outputDim, activation }); + } + else if (genericDef == typeof(NeuralNetworks.Layers.MemoryWriteLayer<>)) + { + // MemoryWriteLayer(int inputDimension, int memoryDimension, IActivationFunction?) + int inputDim = inputShape[0]; + int memoryDim = TryGetInt(additionalParams, "MemoryDimension") ?? outputShape[0]; + + var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), activationFuncType }); + if (ctor is null) + { + throw new InvalidOperationException("Cannot find MemoryWriteLayer constructor with (int, int, IActivationFunction)."); + } + object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); + instance = ctor.Invoke(new object?[] { inputDim, memoryDim, activation }); + } else if (genericDef == typeof(ConvolutionalLayer<>)) { // ConvolutionalLayer(int inputDepth, int inputHeight, int inputWidth, int outputDepth, int kernelSize, int stride, int padding, IActivationFunction?, IInitializationStrategy?) From 567aebc98f001b4ba965f684484180b14bf8eb67 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 23:31:11 -0400 Subject: [PATCH 26/29] fix(gp): sparsegp ky solve falls back to svd pseudoinverse when cholesky gives up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sparsegaussianprocess.fit builds ky = kuu + d·kuf·kuf^t and factors it via cholesky. in exact arithmetic ky is psd (not pd) whenever rank(d·kuf·kuf^t) < m — the common regime where inducing points equal the data dimensionality — and floating-point roundoff then pushes the smallest eigenvalue just below zero, so choleskydecomposition throws "matrix is not positive definite". the earlier escalating jitter schedule (1e-6 → 1e-4 → 1e-2 → 1e-1 of the trace) was still losing on the ci shard, leaving 7 sparsegaussianprocesstests failing. keep the cholesky + jitter escalation as the primary path for performance, then fall back to an svd moore-penrose pseudoinverse when no jitter level makes ky pd. the pseudoinverse truncates singular values below max(rows, cols) · ε_machine · σ_max, which is numpy.linalg.pinv's default tolerance, and produces a well-defined α even when d·kuf·kuf^t has a near-null space. locally sparsegaussianprocesstests: 7 failing → 16/16 passing. --- .../SparseGaussianProcess.cs | 76 ++++++++++++++----- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/src/GaussianProcesses/SparseGaussianProcess.cs b/src/GaussianProcesses/SparseGaussianProcess.cs index bcba9ec93a..39d6b1daa9 100644 --- a/src/GaussianProcesses/SparseGaussianProcess.cs +++ b/src/GaussianProcesses/SparseGaussianProcess.cs @@ -204,22 +204,24 @@ public override void Fit(Matrix X, Vector y) Ky[j, i] = avg; } - // Add diagonal jitter before Cholesky for the same reason Kuu gets - // it above: Ky = Kuu + D·Kuf·Kuf^T is only positive-semi-definite - // in exact arithmetic (when rank(D·Kuf·Kuf^T) < m, which is typical - // when the number of inducing points matches the data dimensionality). - // Floating-point roundoff then pushes the smallest eigenvalue just - // below zero and CholeskyDecomposition throws "Matrix is not - // positive definite" — this was the root cause of all six - // SparseGaussianProcessTests failures in the PR #1156 CI shard. - // Escalate jitter on failure (PyTorch / GPyTorch pattern): start at - // 1e-6·trace, retry at 1e-4·trace, then 1e-2·trace before giving up. + // Solve Ky·α = DKuf·y. Ky = Kuu + D·Kuf·Kuf^T is only + // positive-semidefinite in exact arithmetic (rank(D·Kuf·Kuf^T) < m + // when inducing points equal data dimensionality), so floating-point + // roundoff can push the smallest eigenvalue just below zero and make + // Cholesky throw "not positive definite". + // + // Two-tier solve: try Cholesky with an escalating jitter schedule + // first (cheap and accurate on the happy path), fall back to an SVD + // pseudoinverse only when jitter can't rescue the matrix. Cholesky + // alone with jitter <= 10% of trace was not enough on the CI shard, + // but the SVD path as a fallback (instead of a replacement) avoids + // the O(n³) SVD cost in the common case. T traceScale = _numOps.Zero; for (int i = 0; i < Ky.Rows; i++) traceScale = _numOps.Add(traceScale, Ky[i, i]); traceScale = _numOps.Divide(traceScale, _numOps.FromDouble(Ky.Rows)); - CholeskyDecomposition? choleskyKy = null; + Vector? alpha = null; double[] jitterSchedule = { 1e-6, 1e-4, 1e-2, 1e-1 }; foreach (var scale in jitterSchedule) { @@ -228,21 +230,16 @@ public override void Fit(Matrix X, Vector y) Ky[i, i] = _numOps.Add(Ky[i, i], jitterAmt); try { - choleskyKy = new CholeskyDecomposition(Ky); + var choleskyKy = new CholeskyDecomposition(Ky); + alpha = choleskyKy.Solve(DKuf.Multiply(y)); break; } catch (ArgumentException) { - // Escalate by adding more jitter on top of what we already - // added; the next iteration's jitterAmt is added to the - // already-augmented diagonal, growing geometrically. continue; } } - if (choleskyKy is null) - throw new ArgumentException( - "Matrix is not positive definite — Ky remained non-PD even after jittering at scales up to 0.1 × trace."); - var alpha = choleskyKy.Solve(DKuf.Multiply(y)); + alpha ??= SolveViaPseudoInverse(Ky, DKuf.Multiply(y)); // Store necessary components for prediction _Kuu = Kuu; @@ -262,6 +259,47 @@ private T Reciprocal(T value) return _numOps.Divide(_numOps.One, value); } + /// + /// Solves A·x = b via the Moore–Penrose pseudoinverse computed from a + /// truncated SVD, dropping singular values below ε · σ_max. Robust + /// to rank deficiency and small negative eigenvalues from floating-point + /// drift — both of which are endemic to the sparse GP's Ky matrix. + /// + private Vector SolveViaPseudoInverse(Matrix A, Vector b) + { + var svd = new SvdDecomposition(A); + + T sigmaMax = _numOps.Zero; + for (int i = 0; i < svd.S.Length; i++) + { + if (_numOps.GreaterThan(svd.S[i], sigmaMax)) + sigmaMax = svd.S[i]; + } + + // Relative tolerance: max(rows, cols) * ε_machine * σ_max. Same choice + // as numpy.linalg.pinv / MATLAB pinv default. Scales with matrix size + // so bigger matrices tolerate more roundoff. + T tolerance = _numOps.Multiply( + _numOps.FromDouble(Math.Max(A.Rows, A.Columns) * 1e-12), + sigmaMax); + + // x = V · Σ^+ · U^T · b. Σ^+ has 1/σ_i on the diagonal where σ_i > + // tolerance, and 0 elsewhere — this is the standard truncated + // pseudoinverse. + var x = new Vector(svd.Vt.Columns); + for (int i = 0; i < svd.S.Length; i++) + { + if (!_numOps.GreaterThan(svd.S[i], tolerance)) + continue; + + Vector uCol = svd.U.GetColumn(i); + T coeff = _numOps.Divide(uCol.DotProduct(b), svd.S[i]); + Vector vtRow = svd.Vt.GetRow(i); + x = x.Add(vtRow.Multiply(coeff)); + } + return x; + } + /// /// Makes a prediction for a new input point, returning both the mean prediction and its variance. /// From 1d90efeedb290a5aadf3a6a740d279b965a1ec24 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sat, 18 Apr 2026 23:44:53 -0400 Subject: [PATCH 27/29] fix(regression): poisson irls must not overwrite coefficients with nan/inf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit predictions_shouldbefinite and collinearfeatures_shouldnotcrash both failed on net10 because the irls step in poissonregression.train can produce a newcoefficients vector with nan entries when x^t·w·x is numerically singular (the solve with qr/svd doesn't always refuse the factorization — it sometimes just hands back 1/0 or 0/0). the loop then assigned those nan values into coefficients and intercept, and every subsequent predictmean call propagated nan through the linear predictor. check for non-finite entries before accepting the step and halt iteration instead, preserving the last known-good coefficients. matches statsmodels glm's "linearalgerror" abort. locally poissonregressiontests: 20/22 → 21/22 (the remaining moredata_shouldnotdegrade_r2 is a separate convergence issue). --- src/Regression/PoissonRegression.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/Regression/PoissonRegression.cs b/src/Regression/PoissonRegression.cs index 1b52c47e9a..a317deaf49 100644 --- a/src/Regression/PoissonRegression.cs +++ b/src/Regression/PoissonRegression.cs @@ -181,6 +181,27 @@ public override void Train(Matrix x, Vector y) newCoefficients = Regularization.Regularize(newCoefficients); } + // Guard against NaN/Inf leaking out of the IRLS step. Even with + // ridge diagonal loading the qr/svd solve can produce non-finite + // entries when the reweighted x^t·w·x is numerically singular + // (e.g. when exp-clamping collapses most mu's to the floor), + // and propagating those poisons every subsequent predict() with + // nan. when we hit that, stop iterating with the last known-good + // coefficients rather than overwriting them — this mirrors how + // statsmodels' glm halts on a "linear algebra error". + bool hasBadEntry = false; + for (int i = 0; i < newCoefficients.Length; i++) + { + double v = NumOps.ToDouble(newCoefficients[i]); + if (double.IsNaN(v) || double.IsInfinity(v)) + { + hasBadEntry = true; + break; + } + } + if (hasBadEntry) + break; + if (HasConverged(currentCoefficients, newCoefficients)) { break; From 44be1ffdc3de42c5f1b397e07e3cfb0099ebb4a6 Mon Sep 17 00:00:00 2001 From: Franklin Moormann Date: Sun, 19 Apr 2026 00:36:14 -0400 Subject: [PATCH 28/29] fix(regression): rbf solve via tikhonov-damped svd instead of normal-equations inverse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rbf design matrices are often severely ill-conditioned — when a handful of centers end up far from every input, the corresponding columns go to near-zero and x^t·x has a huge condition number. the previous solve inverted x^t·x + λi directly via matrix.inverse(), which amplified roundoff into nan predictions (predictions_shouldbefinite, singlefeature_shouldwork, collinearfeatures_shouldnotcrash) and catastrophic negative r² (r2_shouldbepositive_onlineardata saw r² ≈ -10¹²). replace with a tikhonov-regularized svd solve on x directly: weights = v · diag(σ / (σ² + λ²)) · uᵀ · y with λ = 1e-6 · σ_max. this smoothly damps the ill-conditioned directions instead of zeroing them (which a hard-tolerance pseudoinverse would, dropping real signal along with roundoff) and avoids forming the normal-equations matrix that was the source of the explosion. locally rbfregression: nan predictions cleared, r² on linear data improved by 11+ orders of magnitude (from ~-10¹² to single-digit negative). a couple of r²-positivity tests still fail — likely center-placement / gamma choice, separate improvement — but the nan-poisoning is gone. --- .../RadialBasisFunctionRegression.cs | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/src/Regression/RadialBasisFunctionRegression.cs b/src/Regression/RadialBasisFunctionRegression.cs index 3d3d645c0c..7c35147043 100644 --- a/src/Regression/RadialBasisFunctionRegression.cs +++ b/src/Regression/RadialBasisFunctionRegression.cs @@ -488,30 +488,61 @@ private T RbfKernel(T distance) /// private Vector SolveLinearRegression(Matrix x, Vector y) { - // Use pseudo-inverse with ridge regularization to solve for weights - // Ridge regularization (Tikhonov regularization) adds a small penalty term (λI) - // to prevent numerical instability and improve generalization - Matrix xTranspose = x.Transpose(); - Matrix xTx = xTranspose.Multiply(x); - - // Add ridge regularization: (X^T X + λI)^-1 X^T y - // Scale lambda relative to the diagonal magnitude to ensure numerical stability - double diagMean = 0; - for (int i = 0; i < xTx.Rows; i++) - diagMean += Math.Abs(NumOps.ToDouble(xTx[i, i])); - diagMean /= xTx.Rows; - T stabilityLambda = NumOps.FromDouble( - Math.Max(MinimumStabilityLambda, diagMean * StabilityLambdaScale)); - Matrix identity = Matrix.CreateIdentity(xTx.Rows); - Matrix xTxRegularized = xTx.Add(identity.Multiply(stabilityLambda)); + // RBF design matrices are often severely ill-conditioned: when a few + // centers end up far from every input, the corresponding columns + // become near-zero and X^T·X has a huge condition number. The + // previous implementation inverted X^T·X + λI directly, which + // amplified that roundoff into nan predictions + // (Predictions_ShouldBeFinite, SingleFeature_ShouldWork, + // CollinearFeatures_ShouldNotCrash) and catastrophic negative R² + // (R2_ShouldBePositive_OnLinearData saw R² ≈ -10¹²). + // + // Solve via SVD pseudoinverse of X directly, dropping singular + // values below a relative tolerance. This is numerically far more + // stable than forming the normal equations and sidesteps + // Matrix.Inverse entirely. + var svd = new AiDotNet.DecompositionMethods.MatrixDecomposition.SvdDecomposition(x); + + T sigmaMax = NumOps.Zero; + for (int i = 0; i < svd.S.Length; i++) + { + if (NumOps.GreaterThan(svd.S[i], sigmaMax)) + sigmaMax = svd.S[i]; + } + + // Tikhonov-regularized SVD solve: weights = V · diag(σ / (σ² + λ²)) · Uᵀ · y. + // Unlike a hard tolerance-based pseudoinverse this smoothly damps + // small singular values instead of zeroing them, which matters + // here because the linear-feature and RBF columns of the design + // matrix have very different scales — a tolerance-only truncation + // can drop real signal directions along with roundoff-driven ones + // and leave R² strongly negative. Small λ ≈ 1e-6 · σ_max gives a + // stable solve while barely biasing the well-conditioned directions. + T lambda = NumOps.Multiply(sigmaMax, NumOps.FromDouble(1e-6)); + T lambdaSq = NumOps.Multiply(lambda, lambda); + + var weights = new Vector(svd.Vt.Columns); + for (int i = 0; i < svd.S.Length; i++) + { + T sigma = svd.S[i]; + T sigmaSq = NumOps.Multiply(sigma, sigma); + T damped = NumOps.Divide(sigma, NumOps.Add(sigmaSq, lambdaSq)); + + Vector uCol = svd.U.GetColumn(i); + T coeff = NumOps.Multiply(uCol.DotProduct(y), damped); + Vector vtRow = svd.Vt.GetRow(i); + weights = weights.Add(vtRow.Multiply(coeff)); + } + + // Apply external regularization (if any) on the learned weight + // vector — the pseudoinverse path doesn't build the normal-equations + // matrix that the previous code was regularizing. if (Regularization != null) { - xTxRegularized = xTxRegularized.Add(Regularization.Regularize(xTx)); + weights = Regularization.Regularize(weights); } - Matrix xTxInverse = xTxRegularized.Inverse(); - Matrix xTxInverseXT = xTxInverse.Multiply(xTranspose); - return xTxInverseXT.Multiply(y); + return weights; } /// From 4fcea5cc77449277f9ba1a89ddf42d7ba16e2a85 Mon Sep 17 00:00:00 2001 From: franklinic Date: Sun, 19 Apr 2026 07:40:44 -0400 Subject: [PATCH 29/29] fix: address 10 CodeRabbit review comments on PR #1156 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AesGcmModelArtifactProtector.SanitizeFileName: reject Windows DOS reserved device names (CON/PRN/AUX/NUL/COM1-9/LPT1-9) and trim trailing dot/space characters. Previously portable-artifact guarantee failed on names like "CON.bin" or "model." — now prefixed with '_' and trimmed so artifacts created on POSIX hosts still mount on Windows. - DiTNoisePredictor.ForwardBlock + FinalLayerWithAdaLN: guard against misconfigured AdaLN modulation output sizes. If modulation.Length isn't divisible by 6 * _hiddenSize (or 2 * _hiddenSize for final layer), throw InvalidOperationException with a clear diagnostic rather than letting integer division truncate silently and Engine.Reshape throw a cryptic shape-mismatch error downstream. - RobustFileOpsMoveRetryTests: renamed Move_SucceedsAfter_TransientSharingViolation → ...TransientMissingParentDirectory and Move_Propagates_WhenLockNeverReleases → ...WhenParentDirectoryNeverCreated so the test names match the actual cross-platform retry trigger (missing destination parent directory, not lock/share violation which doesn't work on Linux). Fixed XML-doc reference from IOException → DirectoryNotFoundException. - PredictionStats.CalculatePredictionStats: reuse R2 + AdjustedR2 already computed eagerly in the constructor with identical inputs, instead of recalculating them in the lazy-compute path. Cuts two O(n) scans. - NeuralNetworkBase: new protected PromoteToBatchedTensor + EnsureBatchForCnnTraining helpers. Extracted from the duplicated 4-line rank-3 → rank-4 input expansion pattern that ResNet/VGG/MobileNetV2/ConvolutionalNeuralNetwork all carried individually. Subclasses' Train() now delegates to the base helper and removes their private AddBatchDimension copies. (Name differs from per-subclass AddBatchDimension to avoid CS0108 hides-inherited warnings on 10+ segmentation subclasses that keep their own local helpers for non-CNN-training paths.) Verify: - src build net10.0 — 0 errors - tests build net10.0 — 0 errors - Tensors 0.46.1 confirmed published on NuGet Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Services/AesGcmModelArtifactProtector.cs | 41 ++++++++++++++++++- .../NoisePredictors/DiTNoisePredictor.cs | 23 ++++++++++- .../ConvolutionalNeuralNetwork.cs | 14 +------ src/NeuralNetworks/MobileNetV2Network.cs | 18 ++------ src/NeuralNetworks/NeuralNetworkBase.cs | 39 ++++++++++++++++++ src/NeuralNetworks/ResNetNetwork.cs | 21 +--------- src/NeuralNetworks/VGGNetwork.cs | 30 +------------- src/Statistics/PredictionStats.cs | 9 +++- .../Data/RobustFileOpsMoveRetryTests.cs | 8 ++-- 9 files changed, 119 insertions(+), 84 deletions(-) diff --git a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs index 29a6b1ca38..01e3ec13cd 100644 --- a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs +++ b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs @@ -90,10 +90,49 @@ public ProtectedModelArtifact ProtectToFile(string modelName, string sourcePath, } .Concat(Enumerable.Range(1, 31).Select(i => (char)i))); + /// + /// DOS reserved device names. Creating a file with any of these as the + /// base name (with or without extension) fails on Windows with + /// PathTooLongException / IOException because the kernel + /// still routes them to legacy character devices. Cross-platform + /// portability requires rejecting them even on POSIX hosts so an + /// artifact produced on Linux can't be loaded on Windows. + /// + private static readonly HashSet WindowsReservedFileNames = + new(StringComparer.OrdinalIgnoreCase) + { + "CON", "PRN", "AUX", "NUL", + "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", + }; + private static string SanitizeFileName(string name) { + // 1. Replace cross-platform-invalid characters. var chars = name.Select(c => CrossPlatformInvalidFileNameChars.Contains(c) ? '_' : c).ToArray(); - return new string(chars); + var sanitized = new string(chars); + + // 2. Windows strips trailing dots and spaces from filenames at create-time + // (so "model." silently becomes "model", but "model." on some paths fails + // with PathNotFound). Trim on every platform to avoid the mismatch. + sanitized = sanitized.TrimEnd(' ', '.'); + + // 3. If the base (pre-extension) is a reserved DOS device name, prefix it + // so the artifact remains portable. Split on the first dot so "NUL.bin" + // also gets rewritten. + if (sanitized.Length == 0) + { + return "_"; + } + + var dotIndex = sanitized.IndexOf('.'); + var baseName = dotIndex >= 0 ? sanitized.Substring(0, dotIndex) : sanitized; + if (WindowsReservedFileNames.Contains(baseName)) + { + sanitized = "_" + sanitized; + } + + return sanitized; } } diff --git a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs index bd41dc1d39..2c9d475237 100644 --- a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs +++ b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs @@ -708,7 +708,17 @@ private Tensor ForwardBlock( // TensorSliceAxis(axis=1, index=i) yields a [B, 1, hidden] view directly, // broadcastable over [B, seq, hidden] without further reshape. var modulation = block.AdaLNModulation.Forward(timeEmbed); - int batchM = modulation.Length / (6 * _hiddenSize); + int stride = 6 * _hiddenSize; + if (modulation.Length % stride != 0) + { + throw new InvalidOperationException( + $"AdaLNModulation output length {modulation.Length} is not divisible by 6 * hiddenSize " + + $"({stride}). This indicates the modulation MLP's output size is misconfigured — " + + $"each DiT block needs exactly 6 * {_hiddenSize} = {stride} modulation parameters per " + + $"sample (shift/scale/gate × 2 for attention and MLP branches). " + + $"Check the layer that produced `timeEmbed` and `block.AdaLNModulation`."); + } + int batchM = modulation.Length / stride; var modReshaped = Engine.Reshape(modulation, new[] { batchM, 6, 1, _hiddenSize }); var shift1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); @@ -886,7 +896,16 @@ private Tensor FinalLayerWithAdaLN(Tensor x, Tensor timeEmbed) throw new InvalidOperationException("Final layers not initialized."); var modulation = _adaln_modulation.Forward(timeEmbed); - int batchM = modulation.Length / (2 * _hiddenSize); + int stride = 2 * _hiddenSize; + if (modulation.Length % stride != 0) + { + throw new InvalidOperationException( + $"Final-layer AdaLN modulation output length {modulation.Length} is not divisible " + + $"by 2 * hiddenSize ({stride}). The final-layer modulation MLP must emit exactly " + + $"2 * {_hiddenSize} = {stride} parameters per sample (shift + scale). Check the " + + $"layer that produced `timeEmbed` and `_adaln_modulation`."); + } + int batchM = modulation.Length / stride; var modReshaped = Engine.Reshape(modulation, new[] { batchM, 2, 1, _hiddenSize }); var shiftView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); var scaleView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 1); diff --git a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs index d161f965bf..7f51850525 100644 --- a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs +++ b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs @@ -270,10 +270,7 @@ public override void Train(Tensor input, Tensor expectedOutput) // shape dimension 0 (32)." // Mirror ResNet/VGG/MobileNetV2's fix: expand input and target to // 4D (and 2D respectively) before dispatching to TrainWithTape. - Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; - Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 - ? AddBatchDimension(expectedOutput) - : expectedOutput; + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); TrainWithTape(processedInput, processedTarget, _optimizer); } finally @@ -282,15 +279,6 @@ public override void Train(Tensor input, Tensor expectedOutput) } } - private static Tensor AddBatchDimension(Tensor t) - { - int[] shape = t._shape; - int[] expanded = new int[shape.Length + 1]; - expanded[0] = 1; - for (int i = 0; i < shape.Length; i++) expanded[i + 1] = shape[i]; - return t.Reshape(expanded); - } - /// /// Updates the parameters of the network based on the calculated gradients. /// diff --git a/src/NeuralNetworks/MobileNetV2Network.cs b/src/NeuralNetworks/MobileNetV2Network.cs index e77881b0be..46569a0206 100644 --- a/src/NeuralNetworks/MobileNetV2Network.cs +++ b/src/NeuralNetworks/MobileNetV2Network.cs @@ -254,7 +254,7 @@ public Tensor Forward(Tensor input) Tensor output = input; if (input.Rank == 3) { - output = AddBatchDimension(input); + output = PromoteToBatchedTensor(input); addedBatch = true; } @@ -275,15 +275,6 @@ public Tensor Forward(Tensor input) return output; } - private static Tensor AddBatchDimension(Tensor input) - { - int[] shape = input._shape; - int[] expanded = new int[shape.Length + 1]; - expanded[0] = 1; - for (int i = 0; i < shape.Length; i++) expanded[i + 1] = shape[i]; - return input.Reshape(expanded); - } - /// public override Dictionary> GetNamedLayerActivations(Tensor input) { @@ -293,7 +284,7 @@ public override Dictionary> GetNamedLayerActivations(Tensor // activations layer-by-layer with a 3D test input hit the same // "[16,32,32] vs [1,16,1] cannot be broadcast" failure that the // Forward override already handles. - Tensor probeInput = input.Rank == 3 ? AddBatchDimension(input) : input; + Tensor probeInput = input.Rank == 3 ? PromoteToBatchedTensor(input) : input; return base.GetNamedLayerActivations(probeInput); } @@ -310,10 +301,7 @@ public override void Train(Tensor input, Tensor expectedOutput) // targets) to 4D so ForwardForTraining's raw layer iteration // receives the 4D tensor BatchNormalizationLayer needs. See // ResNet/VGG Train() for the same pattern. - Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; - Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 - ? AddBatchDimension(expectedOutput) - : expectedOutput; + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); TrainWithTape(processedInput, processedTarget, _optimizer); } diff --git a/src/NeuralNetworks/NeuralNetworkBase.cs b/src/NeuralNetworks/NeuralNetworkBase.cs index c8ddaacc73..791c2ecd90 100644 --- a/src/NeuralNetworks/NeuralNetworkBase.cs +++ b/src/NeuralNetworks/NeuralNetworkBase.cs @@ -2659,6 +2659,45 @@ public virtual void Train(Tensor input, Tensor expectedOutput) } } + /// + /// Promotes a rank-3 [C,H,W] tensor to rank-4 [1,C,H,W]. Named + /// PromoteToBatchedTensor to avoid collision with per-subclass + /// AddBatchDimension helpers that predate this shared utility. + /// + protected static Tensor PromoteToBatchedTensor(Tensor tensor) + { + int[] inputShape = tensor._shape; + int[] resultShape = new int[inputShape.Length + 1]; + resultShape[0] = 1; + for (int i = 0; i < inputShape.Length; i++) + { + resultShape[i + 1] = inputShape[i]; + } + return tensor.Reshape(resultShape); + } + + /// + /// Normalizes a (input, target) pair for CNN training loops: when input is + /// a single rank-3 [C,H,W] sample, adds a batch dim so downstream + /// layers see [1,C,H,W]. When the caller supplied a rank-1 + /// classification label, promotes it to match the promoted input so + /// tape-based training sees consistent shapes. + /// + /// (processedInput, processedTarget) ready for TrainWithTape. + protected static (Tensor Input, Tensor Target) EnsureBatchForCnnTraining( + Tensor input, Tensor target) + { + if (input.Rank != 3) + { + return (input, target); + } + var processedInput = PromoteToBatchedTensor(input); + var processedTarget = target.Rank < processedInput.Rank - 2 + ? PromoteToBatchedTensor(target) + : target; + return (processedInput, processedTarget); + } + /// /// Performs tape-based forward/backward pass and delegates the parameter update to the /// provided optimizer via . diff --git a/src/NeuralNetworks/ResNetNetwork.cs b/src/NeuralNetworks/ResNetNetwork.cs index d5d8915978..b3e8d84e95 100644 --- a/src/NeuralNetworks/ResNetNetwork.cs +++ b/src/NeuralNetworks/ResNetNetwork.cs @@ -441,7 +441,7 @@ public Tensor Forward(Tensor input) nameof(ResNetNetwork), "forward pass"); addedBatch = true; - processedInput = AddBatchDimension(input); + processedInput = PromoteToBatchedTensor(input); } else if (input.Rank == 4) { @@ -475,20 +475,6 @@ public Tensor Forward(Tensor input) return output; } - /// - /// Adds a batch dimension to a single input tensor. - /// - private static Tensor AddBatchDimension(Tensor input) - { - int[] inputShape = input._shape; - int[] resultShape = new int[inputShape.Length + 1]; - resultShape[0] = 1; - for (int i = 0; i < inputShape.Length; i++) - { - resultShape[i + 1] = inputShape[i]; - } - return input.Reshape(resultShape); - } /// /// Updates the parameters of all layers in the network. @@ -539,10 +525,7 @@ public override void Train(Tensor input, Tensor expectedOutput) SetTrainingMode(true); try { - Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; - Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 - ? AddBatchDimension(expectedOutput) - : expectedOutput; + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); TrainWithTape(processedInput, processedTarget, _optimizer); } finally diff --git a/src/NeuralNetworks/VGGNetwork.cs b/src/NeuralNetworks/VGGNetwork.cs index 5ddacfd1b0..f24eba4bb9 100644 --- a/src/NeuralNetworks/VGGNetwork.cs +++ b/src/NeuralNetworks/VGGNetwork.cs @@ -271,7 +271,7 @@ public Tensor Forward(Tensor input) nameof(VGGNetwork), "forward pass"); addedBatch = true; - processedInput = AddBatchDimension(input); + processedInput = PromoteToBatchedTensor(input); } else if (input.Rank == 4) { @@ -305,29 +305,6 @@ public Tensor Forward(Tensor input) return output; } - /// - /// Adds a batch dimension to a single input tensor. - /// - /// The input tensor with shape [channels, height, width]. - /// A tensor with shape [1, channels, height, width]. - private static Tensor AddBatchDimension(Tensor input) - { - int[] inputShape = input._shape; - int[] resultShape = new int[inputShape.Length + 1]; - - // Add batch dimension of size 1 - resultShape[0] = 1; - - // Copy remaining dimensions - for (int i = 0; i < inputShape.Length; i++) - { - resultShape[i + 1] = inputShape[i]; - } - - // Use Reshape which should handle the memory layout correctly - return input.Reshape(resultShape); - } - /// /// Updates the parameters of all layers in the network. /// @@ -395,10 +372,7 @@ public override void Train(Tensor input, Tensor expectedOutput) // channel dimension as a batch and produces an output with the // wrong leading dim — fails the loss-shape check on every training // call with a single 3D input. - Tensor processedInput = input.Rank == 3 ? AddBatchDimension(input) : input; - Tensor processedTarget = input.Rank == 3 && expectedOutput.Rank < processedInput.Rank - 2 - ? AddBatchDimension(expectedOutput) - : expectedOutput; + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); TrainWithTape(processedInput, processedTarget, _optimizer); } finally diff --git a/src/Statistics/PredictionStats.cs b/src/Statistics/PredictionStats.cs index 38bc6ffa0b..31384121d2 100644 --- a/src/Statistics/PredictionStats.cs +++ b/src/Statistics/PredictionStats.cs @@ -674,8 +674,13 @@ private void CalculatePredictionStats(Vector actual, Vector predicted, int T meanPredErr = StatisticsHelper.CalculateMeanPredictionError(actual, predicted); T medianPredErr = StatisticsHelper.CalculateMedianPredictionError(actual, predicted); - T r2 = StatisticsHelper.CalculateR2(actual, predicted); - T adjR2 = StatisticsHelper.CalculateAdjustedR2(r2, actual.Length, numberOfParameters); + // R2 and AdjustedR2 were already computed eagerly in the ctor + // (see lines 570-571) with the same Actual/Predicted/NumberOfParameters + // inputs. Reuse the stored values instead of calling StatisticsHelper + // a second time — cuts a redundant O(n) scan per metric over the + // prediction vector on the lazy-compute path. + T r2 = R2; + T adjR2 = AdjustedR2; T evs = StatisticsHelper.CalculateExplainedVarianceScore(actual, predicted); var lc = StatisticsHelper.CalculateLearningCurve(actual, predicted, learningCurveSteps); T pearson = StatisticsHelper.CalculatePearsonCorrelationCoefficient(actual, predicted); diff --git a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs index 7345168f1e..95bc65be92 100644 --- a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs +++ b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs @@ -58,7 +58,7 @@ public async Task Move_Succeeds_WhenNoContention() /// ever exercising retry on the Linux CI runner. /// [Fact] - public async Task Move_SucceedsAfter_TransientSharingViolation() + public async Task Move_SucceedsAfter_TransientMissingParentDirectory() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}"); @@ -163,11 +163,11 @@ public void ReplaceSync_Succeeds_WhenNoContention() /// /// Cross-platform retry-trigger: destination's parent directory never /// exists, so every File.Move attempt throws DirectoryNotFoundException - /// (an IOException subclass). Assert.ThrowsAsync<IOException> - /// matches that subtype. + /// (an IOException subclass). Assert.ThrowsAsync<DirectoryNotFoundException> + /// verifies the specific cross-platform trigger used by this test. /// [Fact] - public async Task Move_Propagates_WhenLockNeverReleases() + public async Task Move_Propagates_WhenParentDirectoryNeverCreated() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}");