diff --git a/Directory.Packages.props b/Directory.Packages.props index d010125b08..b9ff49519f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ - + diff --git a/src/AiDotNet.Generators/TestScaffoldGenerator.cs b/src/AiDotNet.Generators/TestScaffoldGenerator.cs index 6b0194a23a..4ffd29ee50 100644 --- a/src/AiDotNet.Generators/TestScaffoldGenerator.cs +++ b/src/AiDotNet.Generators/TestScaffoldGenerator.cs @@ -1492,7 +1492,7 @@ private static void EmitGeneratedTestClass( } } } - else if (model.Domains.Contains(3) && !model.Tasks.Contains(35)) + else if (model.Domains.Contains(4) && !model.Tasks.Contains(35)) { // Temporal video models (ActionRecognition=22, VideoGeneration=41, etc.) // need a 4D [frames, channels, height, width] input shape, but @@ -1500,6 +1500,13 @@ private static void EmitGeneratedTestClass( // Rather than silently emit a mismatched 3D architecture alongside a // 4D InputShape, route these to a runtime placeholder until the // architecture type can represent a temporal dimension. + // + // The enum ordinal for ModelDomain.Video is 4 + // (General=0, Vision=1, Language=2, Audio=3, Video=4, ...). + // This check previously used 3, which incorrectly flagged every + // *audio* model (PlayHT, Bark, etc.) as "temporal video" and + // emitted a NotImplementedException factory — ten PlayHTTests + // failures on PR #1156 traced to this off-by-one. constructorExpr = "throw new System.NotImplementedException(" + $"\"'{GeneratorHelpers.StripGenericSuffix(model.ClassName)}' is a temporal video model; NeuralNetworkArchitecture cannot express its 4D [frames, channels, height, width] input. Implement this factory manually.\")"; } @@ -1510,7 +1517,7 @@ private static void EmitGeneratedTestClass( // others default to OneDimensional. Temporal video is handled above. needsArchitectureUsing = true; bool isVision = model.Domains.Contains(1) || model.Domains.Contains(11); // Vision=1, ThreeD=11 - bool isAudio = model.Domains.Contains(4); // Audio=4 + bool isAudio = model.Domains.Contains(3); // Audio=3 (enum ordinal, not Video=4) bool isFrameInterp = model.Tasks.Contains(35); // FrameInterpolation → 3D input string inputTypeExpr; @@ -1623,11 +1630,12 @@ private static void EmitGeneratedTestClass( // Override InputShape/OutputShape for domain-appropriate test data. // Vision/Video/3D models need [C, H, W]; default is [1, 4]. - bool isVideoModel = model.Domains.Contains(3); + // Enum ordinals: General=0, Vision=1, Language=2, Audio=3, Video=4. + bool isVideoModel = model.Domains.Contains(4); // Video=4 (was incorrectly 3) bool isFrameInterpModel = model.Tasks.Contains(35); // FrameInterpolation bool isTemporalVideoModel = isVideoModel && !isFrameInterpModel; bool isVisionModel = model.Domains.Contains(1) || model.Domains.Contains(11); - bool isAudioModel = model.Domains.Contains(4); + bool isAudioModel = model.Domains.Contains(3); // Audio=3 (was incorrectly 4) if (isTemporalVideoModel) { // Temporal video: [frames, channels, height, width] @@ -1660,6 +1668,23 @@ private static void EmitGeneratedTestClass( sb.AppendLine($" protected override int[] InputShape => new[] {{ {dim} }};"); sb.AppendLine(" protected override int[] OutputShape => new[] { 4 };"); } + else if (family == TestFamily.TransformerNER || family == TestFamily.SpanBasedNER) + { + // TransformerNERBase and SpanBasedNERBase both default to + // HiddenDimension=768 (BERT-base). Inputs are validated as + // [seqLen, 768], so the base-class default [1, 4] causes a + // hard "embedding dim mismatch" failure inside MultiHeadAttention + // before any downstream logic runs. Use a short sequence to + // keep the test fast while matching the model's expected + // embedding size. Models with non-default hidden dimensions + // (TinyBERT=312, etc.) need a manual test override. + sb.AppendLine(" protected override int[] InputShape => new[] { 8, 768 };"); + } + else if (family == TestFamily.SequenceLabelingNER) + { + // LSTM-CRF family defaults to EmbeddingDimension=100. + sb.AppendLine(" protected override int[] InputShape => new[] { 8, 100 };"); + } sb.AppendLine($" protected override {returnTypeCode} {factoryMethodName}()"); sb.AppendLine(factoryBody); diff --git a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs index 77be8bffe9..01e3ec13cd 100644 --- a/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs +++ b/src/AiDotNet.Serving/Services/AesGcmModelArtifactProtector.cs @@ -72,11 +72,67 @@ public ProtectedModelArtifact ProtectToFile(string modelName, string sourcePath, } } + /// + /// Cross-platform-invalid filename characters. Combines the Windows + /// invalid set (most restrictive: ":" + "\\" + reserved punctuation + + /// control chars) with POSIX "/" and "\0". Used instead of + /// because that method + /// returns a platform-specific set — on Linux it only contains '\0' + /// and '/', so a model name like "my:model" sanitizes to "my:model" + /// on Linux but "my_model" on Windows. Encrypted artifacts are + /// designed to be portable, so we apply the strict Windows superset + /// on every OS to guarantee the output is mountable everywhere. + /// + private static readonly HashSet CrossPlatformInvalidFileNameChars = + new(new[] + { + '\0', '/', '\\', ':', '*', '?', '"', '<', '>', '|', + } + .Concat(Enumerable.Range(1, 31).Select(i => (char)i))); + + /// + /// DOS reserved device names. Creating a file with any of these as the + /// base name (with or without extension) fails on Windows with + /// PathTooLongException / IOException because the kernel + /// still routes them to legacy character devices. Cross-platform + /// portability requires rejecting them even on POSIX hosts so an + /// artifact produced on Linux can't be loaded on Windows. + /// + private static readonly HashSet WindowsReservedFileNames = + new(StringComparer.OrdinalIgnoreCase) + { + "CON", "PRN", "AUX", "NUL", + "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", + "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", + }; + private static string SanitizeFileName(string name) { - var invalid = Path.GetInvalidFileNameChars(); - var chars = name.Select(c => invalid.Contains(c) ? '_' : c).ToArray(); - return new string(chars); + // 1. Replace cross-platform-invalid characters. + var chars = name.Select(c => CrossPlatformInvalidFileNameChars.Contains(c) ? '_' : c).ToArray(); + var sanitized = new string(chars); + + // 2. Windows strips trailing dots and spaces from filenames at create-time + // (so "model." silently becomes "model", but "model." on some paths fails + // with PathNotFound). Trim on every platform to avoid the mismatch. + sanitized = sanitized.TrimEnd(' ', '.'); + + // 3. If the base (pre-extension) is a reserved DOS device name, prefix it + // so the artifact remains portable. Split on the first dot so "NUL.bin" + // also gets rewritten. + if (sanitized.Length == 0) + { + return "_"; + } + + var dotIndex = sanitized.IndexOf('.'); + var baseName = dotIndex >= 0 ? sanitized.Substring(0, dotIndex) : sanitized; + if (WindowsReservedFileNames.Contains(baseName)) + { + sanitized = "_" + sanitized; + } + + return sanitized; } } diff --git a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs index 55b3139c92..2c9d475237 100644 --- a/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs +++ b/src/Diffusion/NoisePredictors/DiTNoisePredictor.cs @@ -574,8 +574,20 @@ private Tensor Forward(Tensor x, Tensor timeEmbed, Tensor? condition } /// - /// Converts image to patches. + /// Converts image to patches via a single reshape + permute + reshape. /// + /// + /// Equivalent to the nested 6-loop scalar copy this used to be: + /// [B, C, H, W] → reshape [B, C, H/p, p, W/p, p] + /// → permute to [B, H/p, W/p, C, p, p] + /// → reshape [B, numPatches, C·p·p]. + /// All three ops route through , so the permutation + /// runs through the engine's vectorized memcpy kernel instead of a + /// scalar C# nested loop. The two reshape steps are zero-copy views; the + /// permute is the only step that materializes (and only because the + /// downstream requires contiguous + /// input). + /// private Tensor Patchify(Tensor x) { var shape = x._shape; @@ -589,48 +601,23 @@ private Tensor Patchify(Tensor x) var numPatches = numPatchesH * numPatchesW; var patchDim = channels * _patchSize * _patchSize; - var patches = TensorAllocator.Rent(new[] { batch, numPatches, patchDim }); - var patchSpan = patches.AsWritableSpan(); - var xSpan = x.AsSpan(); - - for (int b = 0; b < batch; b++) - { - int patchIdx = 0; - for (int ph = 0; ph < numPatchesH; ph++) - { - for (int pw = 0; pw < numPatchesW; pw++) - { - // Extract patch - int dimIdx = 0; - for (int c = 0; c < channels; c++) - { - for (int py = 0; py < _patchSize; py++) - { - for (int px = 0; px < _patchSize; px++) - { - var ih = ph * _patchSize + py; - var iw = pw * _patchSize + px; - var srcIdx = b * channels * height * width + - c * height * width + - ih * width + iw; - var dstIdx = b * numPatches * patchDim + - patchIdx * patchDim + dimIdx; - patchSpan[dstIdx] = xSpan[srcIdx]; - dimIdx++; - } - } - } - patchIdx++; - } - } - } - - return patches; + // [B, C, H, W] -> [B, C, H/p, p, W/p, p] + var split = Engine.Reshape(x, + new[] { batch, channels, numPatchesH, _patchSize, numPatchesW, _patchSize }); + // -> [B, H/p, W/p, C, p, p] (axes: 0, 2, 4, 1, 3, 5) + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 4, 1, 3, 5 }); + // -> [B, numPatches, patchDim] + return Engine.Reshape(permuted, new[] { batch, numPatches, patchDim }); } /// /// Embeds patches through linear projection using batched forward pass. /// + /// + /// Uses zero-copy views around the dense + /// projection — the previous TensorAllocator.Rent + CopyTo scratch + /// buffers were unnecessary because Patchify's output is contiguous. + /// private Tensor EmbedPatches(Tensor patches) { if (_patchEmbed == null) @@ -641,18 +628,10 @@ private Tensor EmbedPatches(Tensor patches) var numPatches = shape[1]; var patchDim = shape[2]; - // Reshape [batch, numPatches, patchDim] -> [batch*numPatches, patchDim] for batched forward - var flatPatches = TensorAllocator.Rent(new[] { batch * numPatches, patchDim }); - patches.AsSpan().CopyTo(flatPatches.AsWritableSpan()); - - // Single batched forward pass through the linear layer + // [B, numPatches, patchDim] -> [B·numPatches, patchDim] view, batched dense forward, reshape back as view. + var flatPatches = Engine.Reshape(patches, new[] { batch * numPatches, patchDim }); var projected = _patchEmbed.Forward(flatPatches); - - // Reshape back to [batch, numPatches, hiddenSize] - var embedded = TensorAllocator.Rent(new[] { batch, numPatches, _hiddenSize }); - projected.AsSpan().CopyTo(embedded.AsWritableSpan()); - - return embedded; + return Engine.Reshape(projected, new[] { batch, numPatches, _hiddenSize }); } /// @@ -700,6 +679,14 @@ private Tensor CreatePositionEmbedding(int numPatches) /// /// Forward pass through a single DiT block with AdaLN. /// + /// + /// AdaLN modulation tensor is reshaped to [B, 6, 1, hidden] and the six + /// shift/scale/gate parameters are obtained as zero-copy + /// views — no T[] allocations, + /// no scalar fill loops. The (1 + scale) precomputation in AdaLN + /// uses the SIMD primitive instead + /// of a per-element NumOps.Add(NumOps.One, scale[h]) loop. + /// private Tensor ForwardBlock( Tensor x, Tensor timeEmbed, @@ -713,18 +700,33 @@ private Tensor ForwardBlock( throw new InvalidOperationException("Block layers not initialized."); } - // Get AdaLN modulation parameters (shift1, scale1, gate1, shift2, scale2, gate2) + // Get AdaLN modulation parameters (shift1, scale1, gate1, shift2, scale2, gate2). + // AdaLNModulation forward output is hidden*6 elements per batch — derive + // batchM from total length so we don't depend on whether the layer + // returns [B, hidden*6] (rank 2) or [hidden*6] (rank 1, when B=1 and the + // dense layer collapses leading dims). Reshape to [B, 6, 1, hidden] so + // TensorSliceAxis(axis=1, index=i) yields a [B, 1, hidden] view directly, + // broadcastable over [B, seq, hidden] without further reshape. var modulation = block.AdaLNModulation.Forward(timeEmbed); - var modSpan = modulation.AsSpan(); + int stride = 6 * _hiddenSize; + if (modulation.Length % stride != 0) + { + throw new InvalidOperationException( + $"AdaLNModulation output length {modulation.Length} is not divisible by 6 * hiddenSize " + + $"({stride}). This indicates the modulation MLP's output size is misconfigured — " + + $"each DiT block needs exactly 6 * {_hiddenSize} = {stride} modulation parameters per " + + $"sample (shift/scale/gate × 2 for attention and MLP branches). " + + $"Check the layer that produced `timeEmbed` and `block.AdaLNModulation`."); + } + int batchM = modulation.Length / stride; + var modReshaped = Engine.Reshape(modulation, new[] { batchM, 6, 1, _hiddenSize }); - // Split modulation into 6 parts - var modSize = _hiddenSize; - var shift1 = ExtractModulation(modSpan, 0, modSize); - var scale1 = ExtractModulation(modSpan, modSize, modSize); - var gate1 = ExtractModulation(modSpan, modSize * 2, modSize); - var shift2 = ExtractModulation(modSpan, modSize * 3, modSize); - var scale2 = ExtractModulation(modSpan, modSize * 4, modSize); - var gate2 = ExtractModulation(modSpan, modSize * 5, modSize); + var shift1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); + var scale1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 1); + var gate1 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 2); + var shift2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 3); + var scale2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 4); + var gate2 = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 5); // Self-attention with AdaLN var normed = block.Norm1.Forward(x); @@ -751,43 +753,18 @@ private Tensor ForwardBlock( } /// - /// Extracts modulation parameters from combined tensor. + /// Applies adaptive layer normalization (Peebles & Xie, 2023): + /// y = x * (1 + scale) + shift. Both + /// and are [B, 1, hidden] tensor views + /// sliced from the AdaLN modulation tensor — no scratch allocation, no + /// scalar fill. The (1 + scale) precomputation runs through the + /// SIMD kernel. /// - private T[] ExtractModulation(ReadOnlySpan modSpan, int offset, int size) + private Tensor ApplyAdaLN(Tensor x, Tensor scaleView, Tensor shiftView) { - var result = new T[size]; - for (int i = 0; i < size; i++) - { - result[i] = modSpan[offset + i]; - } - return result; - } - - /// - /// Applies adaptive layer normalization using IEngine for hardware acceleration. - /// y = x * (1 + scale) + shift - /// - private Tensor ApplyAdaLN(Tensor x, T[] scale, T[] shift) - { - var shape = x._shape; - var hidden = shape[^1]; - - // Create broadcastable tensors for scale and shift: [1, 1, hidden] - var scaleTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var shiftTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var scaleSpan = scaleTensor.AsWritableSpan(); - var shiftSpan = shiftTensor.AsWritableSpan(); - - for (int h = 0; h < hidden; h++) - { - scaleSpan[h] = NumOps.Add(NumOps.One, scale[h % scale.Length]); - shiftSpan[h] = shift[h % shift.Length]; - } - - // y = x * (1 + scale) + shift — per DiT paper (Peebles & Xie, 2023) - // scale/shift are [1,1,hidden], x is [batch, seq, hidden] — requires broadcasting - var scaled = Engine.TensorBroadcastMultiply(x, scaleTensor); - return Engine.TensorBroadcastAdd(scaled, shiftTensor); + var scalePlusOne = Engine.TensorAddScalar(scaleView, NumOps.One); + var scaled = Engine.TensorBroadcastMultiply(x, scalePlusOne); + return Engine.TensorBroadcastAdd(scaled, shiftView); } /// @@ -861,166 +838,117 @@ private Tensor ApplyCrossAttention( } /// - /// Reshapes tensor from [batch, seq, hidden] to [batch*heads, seq, headDim] for multi-head attention. + /// Reshapes tensor from [batch, seq, hidden] to + /// [batch*heads, seq, headDim] for multi-head attention via a + /// reshape + permute + reshape pipeline (no scalar nested copy loops). /// - private static Tensor ReshapeForHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) + private Tensor ReshapeForHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) { - var result = TensorAllocator.Rent(new[] { batch * numHeads, seq, headDim }); - var srcSpan = tensor.AsSpan(); - var dstSpan = result.AsWritableSpan(); - var hidden = numHeads * headDim; - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < numHeads; h++) - { - for (int s = 0; s < seq; s++) - { - var srcBase = b * seq * hidden + s * hidden + h * headDim; - var dstBase = (b * numHeads + h) * seq * headDim + s * headDim; - srcSpan.Slice(srcBase, headDim).CopyTo(dstSpan.Slice(dstBase, headDim)); - } - } - } - - return result; + // [B, S, H·D] -> [B, S, H, D] + var split = Engine.Reshape(tensor, new[] { batch, seq, numHeads, headDim }); + // -> [B, H, S, D] + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 1, 3 }); + // -> [B·H, S, D] + return Engine.Reshape(permuted, new[] { batch * numHeads, seq, headDim }); } /// - /// Reshapes tensor from [batch*heads, seq, headDim] back to [batch, seq, hidden]. + /// Inverse of — collapses head and batch + /// back together via reshape + permute + reshape. /// - private static Tensor ReshapeFromHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) + private Tensor ReshapeFromHeads(Tensor tensor, int batch, int seq, int numHeads, int headDim) { - var result = TensorAllocator.Rent(new[] { batch, seq, numHeads * headDim }); - var srcSpan = tensor.AsSpan(); - var dstSpan = result.AsWritableSpan(); - var hidden = numHeads * headDim; - - for (int b = 0; b < batch; b++) - { - for (int h = 0; h < numHeads; h++) - { - for (int s = 0; s < seq; s++) - { - var srcBase = (b * numHeads + h) * seq * headDim + s * headDim; - var dstBase = b * seq * hidden + s * hidden + h * headDim; - srcSpan.Slice(srcBase, headDim).CopyTo(dstSpan.Slice(dstBase, headDim)); - } - } - } - - return result; + // [B·H, S, D] -> [B, H, S, D] + var split = Engine.Reshape(tensor, new[] { batch, numHeads, seq, headDim }); + // -> [B, S, H, D] + var permuted = Engine.TensorPermute(split, new[] { 0, 2, 1, 3 }); + // -> [B, S, H·D] + return Engine.Reshape(permuted, new[] { batch, seq, numHeads * headDim }); } /// - /// Adds with gating using IEngine for hardware acceleration. - /// result = x + gate * residual + /// Gated residual add: result = x + gateView * residual. + /// is a [B, 1, hidden] view sliced from + /// the AdaLN modulation tensor — no scratch allocation, no scalar fill. + /// Uses for the per-channel + /// gate broadcast. /// - private Tensor AddWithGate(Tensor x, Tensor residual, T[] gate) + private Tensor AddWithGate(Tensor x, Tensor residual, Tensor gateView) { - var hidden = x.Shape[^1]; - - // Create broadcastable gate tensor: [1, 1, hidden] - var gateTensor = TensorAllocator.Rent(new[] { 1, 1, hidden }); - var gateSpan = gateTensor.AsWritableSpan(); - for (int h = 0; h < hidden; h++) - { - gateSpan[h] = gate[h % gate.Length]; - } - - // result = x + gate * residual - var gated = Engine.TensorMultiply(residual, gateTensor); + var gated = Engine.TensorBroadcastMultiply(residual, gateView); return Engine.TensorAdd(x, gated); } /// - /// Final layer with AdaLN zero using batched forward pass. + /// Final layer with AdaLN-zero using batched forward pass. /// + /// + /// Modulation slicing matches : reshape to + /// [B, 2, 1, hidden] then for + /// shift/scale views. Reshape between the projection input and output uses + /// views instead of + /// TensorAllocator.Rent+CopyTo scratch buffers, eliminating + /// two allocations per inference step. + /// private Tensor FinalLayerWithAdaLN(Tensor x, Tensor timeEmbed) { if (_finalNorm == null || _adaln_modulation == null || _outputProj == null) throw new InvalidOperationException("Final layers not initialized."); - // Get final modulation (scale, shift) var modulation = _adaln_modulation.Forward(timeEmbed); - var modSpan = modulation.AsSpan(); - - var shift = ExtractModulation(modSpan, 0, _hiddenSize); - var scale = ExtractModulation(modSpan, _hiddenSize, _hiddenSize); + int stride = 2 * _hiddenSize; + if (modulation.Length % stride != 0) + { + throw new InvalidOperationException( + $"Final-layer AdaLN modulation output length {modulation.Length} is not divisible " + + $"by 2 * hiddenSize ({stride}). The final-layer modulation MLP must emit exactly " + + $"2 * {_hiddenSize} = {stride} parameters per sample (shift + scale). Check the " + + $"layer that produced `timeEmbed` and `_adaln_modulation`."); + } + int batchM = modulation.Length / stride; + var modReshaped = Engine.Reshape(modulation, new[] { batchM, 2, 1, _hiddenSize }); + var shiftView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 0); + var scaleView = Engine.TensorSliceAxis(modReshaped, axis: 1, index: 1); - // Apply final norm with AdaLN var normed = _finalNorm.Forward(x); - normed = ApplyAdaLN(normed, scale, shift); + normed = ApplyAdaLN(normed, scaleView, shiftView); - // Project to output dimension using batched forward pass var shape = normed._shape; var batch = shape[0]; var numPatches = shape[1]; var patchDim = _inputChannels * _patchSize * _patchSize; - // Reshape [batch, numPatches, hiddenSize] -> [batch*numPatches, hiddenSize] - var flatNormed = TensorAllocator.Rent(new[] { batch * numPatches, _hiddenSize }); - normed.AsSpan().CopyTo(flatNormed.AsWritableSpan()); - - // Single batched forward pass + // [batch, numPatches, hiddenSize] -> [batch*numPatches, hiddenSize] via view. + var flatNormed = Engine.Reshape(normed, new[] { batch * numPatches, _hiddenSize }); var projected = _outputProj.Forward(flatNormed); - - // Reshape back to [batch, numPatches, patchDim] - var output = TensorAllocator.Rent(new[] { batch, numPatches, patchDim }); - projected.AsSpan().CopyTo(output.AsWritableSpan()); - - return output; + // Back to [batch, numPatches, patchDim] via view. + return Engine.Reshape(projected, new[] { batch, numPatches, patchDim }); } /// - /// Converts patches back to image. + /// Inverse of — reconstructs the spatial image. /// + /// + /// [B, numPatches, C·p·p] → reshape [B, H/p, W/p, C, p, p] + /// → permute to [B, C, H/p, p, W/p, p] + /// → reshape [B, C, H, W]. Same vectorized-memcpy + view pattern + /// as — zero scalar loops, one materialization. + /// private Tensor Unpatchify(Tensor patches, int height, int width) { var shape = patches._shape; var batch = shape[0]; - var patchDim = shape[2]; var numPatchesH = height / _patchSize; var numPatchesW = width / _patchSize; - var output = TensorAllocator.Rent(new[] { batch, _inputChannels, height, width }); - var outputSpan = output.AsWritableSpan(); - var patchSpan = patches.AsSpan(); - - for (int b = 0; b < batch; b++) - { - int patchIdx = 0; - for (int ph = 0; ph < numPatchesH; ph++) - { - for (int pw = 0; pw < numPatchesW; pw++) - { - // Reconstruct patch - int dimIdx = 0; - for (int c = 0; c < _inputChannels; c++) - { - for (int py = 0; py < _patchSize; py++) - { - for (int px = 0; px < _patchSize; px++) - { - var ih = ph * _patchSize + py; - var iw = pw * _patchSize + px; - var dstIdx = b * _inputChannels * height * width + - c * height * width + - ih * width + iw; - var srcIdx = b * numPatchesH * numPatchesW * patchDim + - patchIdx * patchDim + dimIdx; - outputSpan[dstIdx] = patchSpan[srcIdx]; - dimIdx++; - } - } - } - patchIdx++; - } - } - } - - return output; + // [B, numPatches, C·p·p] -> [B, H/p, W/p, C, p, p] + var unsplit = Engine.Reshape(patches, + new[] { batch, numPatchesH, numPatchesW, _inputChannels, _patchSize, _patchSize }); + // -> [B, C, H/p, p, W/p, p] (inverse permutation: axes 0, 3, 1, 4, 2, 5) + var permuted = Engine.TensorPermute(unsplit, new[] { 0, 3, 1, 4, 2, 5 }); + // -> [B, C, H, W] + return Engine.Reshape(permuted, new[] { batch, _inputChannels, height, width }); } /// diff --git a/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs b/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs index 3a753b1e14..21408f1740 100644 --- a/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs +++ b/src/Diffusion/NoisePredictors/VideoUNetPredictor.cs @@ -1019,7 +1019,28 @@ private Tensor StackFrames(List> frames) private ILayer CreateSpatialResBlock(int inChannels, int outChannels) { - return LazyDense(inChannels, outChannels, new SiLUActivation()); + // A diffusion spatial ResBlock transforms the channel dimension of a + // 4D [B, C, H, W] feature map. The previous implementation used + // LazyDense(inChannels, outChannels), but DenseLayer projects the + // *last* dimension of its input — for a 4D tensor that is the width + // axis, not channels. The block therefore left C unchanged while + // scrambling W to outChannels, and every subsequent TimeCondProjection + // (sized for the planned outChannels) saw a feature map still at the + // incoming channel count — hence "FiLM conditioning projection width + // mismatch: expected 640, got 1280" on upscaleavideomodel and + // streamingt2vmodel tests. + // A 1x1 Conv2D is the standard channel-mixing primitive: it consumes + // [B, inChannels, H, W] and produces [B, outChannels, H, W] while + // leaving spatial dims alone. + return LazyConv2D( + inputDepth: inChannels, + inputHeight: 1, + inputWidth: 1, + outputDepth: outChannels, + kernelSize: 1, + stride: 1, + padding: 0, + activation: new SiLUActivation()); } /// diff --git a/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs b/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs index d335168816..b0e8429596 100644 --- a/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs +++ b/src/Diffusion/SuperResolution/UpscaleAVideoModel.cs @@ -134,13 +134,20 @@ public class UpscaleAVideoModel : VideoDiffusionModelBase private const int CROSS_ATTENTION_DIM = 1024; /// - /// Input channels for the Video U-Net (8 = 4 latent + 4 low-res conditioning). + /// Input channels for the Video U-Net input convolution (4 = latent channels). /// /// - /// The Video U-Net receives concatenated latent noise and downscaled low-resolution - /// video frames as conditioning, doubling the standard 4 latent channels. + /// The reference implementation describes this model as using 8 input channels + /// (4 latent + 4 downscaled low-res conditioning), but that design expects the + /// conditioning to be concatenated with the latents before the first conv. + /// Our VideoUNetPredictor.ForwardVideoUNet adds the image condition via + /// _imageCondProjection *after* the input conv instead — so the input conv + /// sees only the latent channels. Pinning INPUT_CHANNELS to LATENT_CHANNELS + /// keeps the conv weight shape consistent with what ForwardVideoUNet actually + /// feeds it, fixing the "Expected input depth 8, but got 4" crash in the + /// UpscaleAVideoModel tests. /// - private const int INPUT_CHANNELS = 8; + private const int INPUT_CHANNELS = LATENT_CHANNELS; /// /// Base channel count for the Video U-Net (320). diff --git a/src/GaussianProcesses/SparseGaussianProcess.cs b/src/GaussianProcesses/SparseGaussianProcess.cs index 520370d485..39d6b1daa9 100644 --- a/src/GaussianProcesses/SparseGaussianProcess.cs +++ b/src/GaussianProcesses/SparseGaussianProcess.cs @@ -204,8 +204,42 @@ public override void Fit(Matrix X, Vector y) Ky[j, i] = avg; } - var choleskyKy = new CholeskyDecomposition(Ky); - var alpha = choleskyKy.Solve(DKuf.Multiply(y)); + // Solve Ky·α = DKuf·y. Ky = Kuu + D·Kuf·Kuf^T is only + // positive-semidefinite in exact arithmetic (rank(D·Kuf·Kuf^T) < m + // when inducing points equal data dimensionality), so floating-point + // roundoff can push the smallest eigenvalue just below zero and make + // Cholesky throw "not positive definite". + // + // Two-tier solve: try Cholesky with an escalating jitter schedule + // first (cheap and accurate on the happy path), fall back to an SVD + // pseudoinverse only when jitter can't rescue the matrix. Cholesky + // alone with jitter <= 10% of trace was not enough on the CI shard, + // but the SVD path as a fallback (instead of a replacement) avoids + // the O(n³) SVD cost in the common case. + T traceScale = _numOps.Zero; + for (int i = 0; i < Ky.Rows; i++) + traceScale = _numOps.Add(traceScale, Ky[i, i]); + traceScale = _numOps.Divide(traceScale, _numOps.FromDouble(Ky.Rows)); + + Vector? alpha = null; + double[] jitterSchedule = { 1e-6, 1e-4, 1e-2, 1e-1 }; + foreach (var scale in jitterSchedule) + { + T jitterAmt = _numOps.Multiply(traceScale, _numOps.FromDouble(scale)); + for (int i = 0; i < Ky.Rows; i++) + Ky[i, i] = _numOps.Add(Ky[i, i], jitterAmt); + try + { + var choleskyKy = new CholeskyDecomposition(Ky); + alpha = choleskyKy.Solve(DKuf.Multiply(y)); + break; + } + catch (ArgumentException) + { + continue; + } + } + alpha ??= SolveViaPseudoInverse(Ky, DKuf.Multiply(y)); // Store necessary components for prediction _Kuu = Kuu; @@ -225,6 +259,47 @@ private T Reciprocal(T value) return _numOps.Divide(_numOps.One, value); } + /// + /// Solves A·x = b via the Moore–Penrose pseudoinverse computed from a + /// truncated SVD, dropping singular values below ε · σ_max. Robust + /// to rank deficiency and small negative eigenvalues from floating-point + /// drift — both of which are endemic to the sparse GP's Ky matrix. + /// + private Vector SolveViaPseudoInverse(Matrix A, Vector b) + { + var svd = new SvdDecomposition(A); + + T sigmaMax = _numOps.Zero; + for (int i = 0; i < svd.S.Length; i++) + { + if (_numOps.GreaterThan(svd.S[i], sigmaMax)) + sigmaMax = svd.S[i]; + } + + // Relative tolerance: max(rows, cols) * ε_machine * σ_max. Same choice + // as numpy.linalg.pinv / MATLAB pinv default. Scales with matrix size + // so bigger matrices tolerate more roundoff. + T tolerance = _numOps.Multiply( + _numOps.FromDouble(Math.Max(A.Rows, A.Columns) * 1e-12), + sigmaMax); + + // x = V · Σ^+ · U^T · b. Σ^+ has 1/σ_i on the diagonal where σ_i > + // tolerance, and 0 elsewhere — this is the standard truncated + // pseudoinverse. + var x = new Vector(svd.Vt.Columns); + for (int i = 0; i < svd.S.Length; i++) + { + if (!_numOps.GreaterThan(svd.S[i], tolerance)) + continue; + + Vector uCol = svd.U.GetColumn(i); + T coeff = _numOps.Divide(uCol.DotProduct(b), svd.S[i]); + Vector vtRow = svd.Vt.GetRow(i); + x = x.Add(vtRow.Multiply(coeff)); + } + return x; + } + /// /// Makes a prediction for a new input point, returning both the mean prediction and its variance. /// diff --git a/src/Helpers/DeserializationHelper.cs b/src/Helpers/DeserializationHelper.cs index 2f16126167..382245d0d7 100644 --- a/src/Helpers/DeserializationHelper.cs +++ b/src/Helpers/DeserializationHelper.cs @@ -466,6 +466,40 @@ public static ILayer CreateLayerFromType(string layerType, int[] inputShap { instance = CreateMultiLoRAAdapter(type, inputShape, outputShape, additionalParams); } + else if (genericDef == typeof(NeuralNetworks.Layers.MemoryReadLayer<>)) + { + // MemoryReadLayer(int inputDimension, int memoryDimension, int outputDimension, IActivationFunction?) + // memoryDimension is a free parameter — the default MemoryNetwork wires + // input == memory == output == embeddingSize, so fall back to output size + // if the serialized metadata doesn't pin it explicitly. + int inputDim = inputShape[0]; + int outputDim = outputShape[0]; + int memoryDim = TryGetInt(additionalParams, "MemoryDimension") ?? outputDim; + + var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType }); + if (ctor is null) + { + throw new InvalidOperationException("Cannot find MemoryReadLayer constructor with (int, int, int, IActivationFunction)."); + } + object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); + instance = ctor.Invoke(new object?[] { inputDim, memoryDim, outputDim, activation }); + } + else if (genericDef == typeof(NeuralNetworks.Layers.MemoryWriteLayer<>)) + { + // MemoryWriteLayer(int inputDimension, int memoryDimension, IActivationFunction?) + int inputDim = inputShape[0]; + int memoryDim = TryGetInt(additionalParams, "MemoryDimension") ?? outputShape[0]; + + var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), activationFuncType }); + if (ctor is null) + { + throw new InvalidOperationException("Cannot find MemoryWriteLayer constructor with (int, int, IActivationFunction)."); + } + object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); + instance = ctor.Invoke(new object?[] { inputDim, memoryDim, activation }); + } else if (genericDef == typeof(ConvolutionalLayer<>)) { // ConvolutionalLayer(int inputDepth, int inputHeight, int inputWidth, int outputDepth, int kernelSize, int stride, int padding, IActivationFunction?, IInitializationStrategy?) @@ -989,7 +1023,16 @@ private static object CreateDenseLayer(Type type, int[] inputShape, int[] out private static object CreateMultiHeadAttentionLayer(Type type, int[] inputShape, Dictionary? additionalParams) { - // MultiHeadAttentionLayer(int sequenceLength, int embeddingDimension, int headCount, IActivationFunction? activationFunction = null) + // MultiHeadAttentionLayer(int sequenceLength, int embeddingDimension, + // int headCount, IActivationFunction? activationFunction = null, + // IInitializationStrategy? initializationStrategy = null) + // + // The optional initializationStrategy parameter MUST be included in + // the reflection signature even though it has a default — Type.GetConstructor + // matches by exact parameter list, not by "first N + defaults". Without it + // the lookup returns null and Clone-via-serialization (used by + // InferenceOptimizer.OptimizeForInference and any model save/load + // round-trip) fails on every transformer with an MHA layer. if (inputShape.Length < 2) { throw new InvalidOperationException("MultiHeadAttentionLayer requires input shape [sequenceLength, embeddingDimension]."); @@ -1000,14 +1043,15 @@ private static object CreateMultiHeadAttentionLayer(Type type, int[] inputSha int headCount = TryGetInt(additionalParams, "HeadCount") ?? ResolveDefaultHeadCount(embDim); var activationFuncType = typeof(IActivationFunction<>).MakeGenericType(typeof(T)); - var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType }); + var initStrategyType = typeof(IInitializationStrategy<>).MakeGenericType(typeof(T)); + var ctor = type.GetConstructor(new Type[] { typeof(int), typeof(int), typeof(int), activationFuncType, initStrategyType }); if (ctor is null) { - throw new InvalidOperationException("Cannot find MultiHeadAttentionLayer constructor with (int, int, int, IActivationFunction)."); + throw new InvalidOperationException("Cannot find MultiHeadAttentionLayer constructor with (int, int, int, IActivationFunction, IInitializationStrategy)."); } object? activation = TryCreateActivationInstance(additionalParams, "ScalarActivationType", activationFuncType); - return ctor.Invoke(new object?[] { seqLen, embDim, headCount, activation }); + return ctor.Invoke(new object?[] { seqLen, embDim, headCount, activation, null }); } private static object CreateFlashAttentionLayer(Type type, int[] inputShape, Dictionary? additionalParams) diff --git a/src/Helpers/LayerHelper.cs b/src/Helpers/LayerHelper.cs index b3b537a83c..6ab75ca5f4 100644 --- a/src/Helpers/LayerHelper.cs +++ b/src/Helpers/LayerHelper.cs @@ -2579,11 +2579,18 @@ public static IEnumerable> CreateDefaultRNNLayers(NeuralNetworkArchite // RNN layers output [seqLen, hiddenSize], but Dense layer expects [hiddenSize] yield return new SequenceLastLayer(hiddenSize); - // Add the final Dense Layer to map to output size + // Add the final Dense Layer to map to output size. + // Use an explicit IdentityActivation so the Dense output keeps its sign — + // passing activationFunction:null falls through to DenseLayer's ReLU + // default, which clipped all regression outputs at zero. With ReLU + // the 2-layer tanh stack produced small mixed-sign values that the + // final Dense then zeroed, so ScaledInput_ShouldChangeOutput saw + // output=0 for both the raw and the 10x input and reported "forward + // pass may ignore input values". yield return new DenseLayer( inputSize: hiddenSize, outputSize: outputSize, - activationFunction: null + activationFunction: new IdentityActivation() ); // Task-appropriate final activation @@ -2717,11 +2724,16 @@ public static IEnumerable> CreateDefaultQuantumNetworkLayers( yield return new QuantumLayer(hiddenSize, quantumDim, numQubits); yield return new MeasurementLayer(quantumDim); - // Final dense layer to map to output size + // Final dense layer to map to output size. Use identity so the + // task-appropriate activation below owns the output non-linearity + // — passing null falls through to DenseLayer's ReLU default and + // clips the pre-activation logits, which collapsed + // scaledinput_shouldchangeoutput / differentinputs for quantum + // regression models. yield return new DenseLayer( inputSize: quantumDim, outputSize: outputSize, - activationFunction: null + activationFunction: new IdentityActivation() ); // Final activation based on task type @@ -4207,11 +4219,23 @@ public static IEnumerable> CreateDefaultUNet3DLayers( // ============== DECODER PATH ============== // Each decoder block: Upsample3D -> Conv3D -> Conv3D - // Note: Skip connections need to be handled by the network model + // + // Note: a full U-Net would concatenate encoder skip-connections at + // each decoder level, doubling the channel count into the first + // Conv3D. This implementation does NOT actually perform the + // concatenation, so the "*2" previously applied to + // encoderFilters[block + 1] told the Upsample and first-Conv3D to + // expect twice the channels they would actually receive from the + // preceding decoder block's Second-Conv3D output. That produced + // "Input channels (128) must match kernel in_channels (256)" + // at the first decoder block after the bottleneck-adjacent one. + // Matching the actual tensor-channel count (= encoderFilters[block + 1]) + // keeps the stack consistent; adding real skip concatenation is a + // separate architectural improvement tracked independently. for (int block = numEncoderBlocks - 2; block >= 0; block--) { int outputFilters = encoderFilters[block]; - int inChannels = block == numEncoderBlocks - 2 ? bottleneckFilters : encoderFilters[block + 1] * 2; + int inChannels = block == numEncoderBlocks - 2 ? bottleneckFilters : encoderFilters[block + 1]; // Upsample3D to increase resolution yield return new Upsample3DLayer( diff --git a/src/Initialization/InitializationStrategyBase.cs b/src/Initialization/InitializationStrategyBase.cs index e483a4ac3f..45e3657f18 100644 --- a/src/Initialization/InitializationStrategyBase.cs +++ b/src/Initialization/InitializationStrategyBase.cs @@ -118,26 +118,15 @@ protected void XavierNormalInitialize(Tensor weights, int fanIn, int fanOut) if (typeof(T) == typeof(double)) { - for (int i = 0; i < span.Length; i++) - { - double value; - do { value = SampleGaussian(0, stddev); } - while (Math.Abs(value) > clipBound); - span[i] = System.Runtime.CompilerServices.Unsafe.As(ref value); - } + var rawArr = (double[])(object)weights.GetDataArray(); + XavierFillDouble(rawArr, 0, weights.Length, stddev, clipBound); return; } if (typeof(T) == typeof(float)) { - for (int i = 0; i < span.Length; i++) - { - double value; - do { value = SampleGaussian(0, stddev); } - while (Math.Abs(value) > clipBound); - float fv = (float)value; - span[i] = System.Runtime.CompilerServices.Unsafe.As(ref fv); - } + var rawArr = (float[])(object)weights.GetDataArray(); + XavierFillFloat(rawArr, 0, weights.Length, stddev, clipBound); return; } @@ -259,4 +248,159 @@ protected double SampleGaussian(double mean, double stddev) var randStdNormal = Math.Sqrt(-2.0 * Math.Log(u1)) * Math.Sin(2.0 * Math.PI * u2); return mean + stddev * randStdNormal; } + + /// + /// Fills a span with N(0, stddev) samples clipped to ±, + /// using a paired Box-Muller transform that produces two samples per pair of uniform + /// draws — halves the / call count vs. + /// calling per element. + /// + /// + /// Replaces the per-element while (Math.Abs(value) > clipBound) do ... + /// rejection loop which was the dominant cost of DiT-XL lazy weight init (each + /// block's Dense / SelfAttention layer paid 1–30 s of RNG overhead on first + /// forward). Rejection rate at 2σ is ~5 %, so in the common case each iteration + /// produces two usable samples with one log + one sqrt + one sin + one cos + two + /// multiplies. The inner loop is a tight unvirtualized local function so JIT can + /// keep everything in registers and auto-vectorize the clip check. + /// + private void XavierFillDouble(double[] dst, int offset, int length, double stddev, double clipBound) + { + if (length == 0) return; + + const int ParallelThreshold = 1 << 18; // 256K doubles ≈ 2MB + int cores = Math.Max(1, Environment.ProcessorCount); + + if (length < ParallelThreshold || cores == 1) + { + FillChunkDouble(dst.AsSpan(offset, length), stddev, clipBound, Random); + return; + } + + // For large tensors (typical DiT-XL hidden×4 ≈ 100M elements), partition + // across cores so init amortizes over the thread pool instead of running + // single-threaded. Pre-seed per-chunk RNGs from the master so the parallel + // work remains deterministic relative to the master seed. System.Random + // is NOT thread-safe, so we MUST use per-thread instances. + int chunkSize = (length + cores - 1) / cores; + var seeds = new int[cores]; + for (int c = 0; c < cores; c++) seeds[c] = Random.Next(); + + System.Threading.Tasks.Parallel.For(0, cores, c => + { + int chunkStart = c * chunkSize; + int chunkEnd = Math.Min(chunkStart + chunkSize, length); + if (chunkStart >= chunkEnd) return; + var chunkRng = new Random(seeds[c]); + FillChunkDouble(dst.AsSpan(offset + chunkStart, chunkEnd - chunkStart), stddev, clipBound, chunkRng); + }); + } + + /// + /// Sequential Box-Muller fill of a span — inner helper used by both the + /// sequential fast path and the parallel chunk workers. + /// + private static void FillChunkDouble(Span dst, double stddev, double clipBound, Random rng) + { + double z1 = 0; + bool havePending = false; + + for (int i = 0; i < dst.Length; i++) + { + double sample; + while (true) + { + if (havePending) + { + sample = z1; + havePending = false; + } + else + { + double u1 = 1.0 - rng.NextDouble(); + double u2 = rng.NextDouble(); + double r = Math.Sqrt(-2.0 * Math.Log(u1)); + double theta = 2.0 * Math.PI * u2; + sample = r * Math.Sin(theta); + z1 = r * Math.Cos(theta); + havePending = true; + } + sample *= stddev; + if (!(sample > clipBound) && !(sample < -clipBound)) + { + dst[i] = sample; + break; + } + havePending = false; + } + } + } + + /// + /// Float variant of . Uses double-precision + /// Box-Muller internally (accuracy matters more than the tiny cost) and + /// narrows to float on store. + /// + private void XavierFillFloat(float[] dst, int offset, int length, double stddev, double clipBound) + { + if (length == 0) return; + + const int ParallelThreshold = 1 << 18; + int cores = Math.Max(1, Environment.ProcessorCount); + + if (length < ParallelThreshold || cores == 1) + { + FillChunkFloat(dst.AsSpan(offset, length), stddev, clipBound, Random); + return; + } + + int chunkSize = (length + cores - 1) / cores; + var seeds = new int[cores]; + for (int c = 0; c < cores; c++) seeds[c] = Random.Next(); + + System.Threading.Tasks.Parallel.For(0, cores, c => + { + int chunkStart = c * chunkSize; + int chunkEnd = Math.Min(chunkStart + chunkSize, length); + if (chunkStart >= chunkEnd) return; + var chunkRng = new Random(seeds[c]); + FillChunkFloat(dst.AsSpan(offset + chunkStart, chunkEnd - chunkStart), stddev, clipBound, chunkRng); + }); + } + + private static void FillChunkFloat(Span dst, double stddev, double clipBound, Random rng) + { + double z1 = 0; + bool havePending = false; + + for (int i = 0; i < dst.Length; i++) + { + double sample; + while (true) + { + if (havePending) + { + sample = z1; + havePending = false; + } + else + { + double u1 = 1.0 - rng.NextDouble(); + double u2 = rng.NextDouble(); + double r = Math.Sqrt(-2.0 * Math.Log(u1)); + double theta = 2.0 * Math.PI * u2; + sample = r * Math.Sin(theta); + z1 = r * Math.Cos(theta); + havePending = true; + } + sample *= stddev; + if (!(sample > clipBound) && !(sample < -clipBound)) + { + dst[i] = (float)sample; + break; + } + havePending = false; + } + } + } } diff --git a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs index da125b4bae..7f51850525 100644 --- a/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs +++ b/src/NeuralNetworks/ConvolutionalNeuralNetwork.cs @@ -260,7 +260,18 @@ public override void Train(Tensor input, Tensor expectedOutput) SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + // Accept both 3D [C, H, W] and 4D [B, C, H, W] inputs — matches the + // Forward contract. ForwardForTraining runs the raw tensor through + // every layer without adjusting rank, so a 3D input would make the + // final FlattenLayer treat the 32-channel dimension as a batch + // (flatten preserves dim 0), producing a [32, 10] prediction + // against a [10] one-hot target — fails the loss shape check: + // "Target shape dimension 0 (10) does not match predicted + // shape dimension 0 (32)." + // Mirror ResNet/VGG/MobileNetV2's fix: expand input and target to + // 4D (and 2D respectively) before dispatching to TrainWithTape. + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { diff --git a/src/NeuralNetworks/Layers/SparseLinearLayer.cs b/src/NeuralNetworks/Layers/SparseLinearLayer.cs index 88f7d95f1f..d3076952ff 100644 --- a/src/NeuralNetworks/Layers/SparseLinearLayer.cs +++ b/src/NeuralNetworks/Layers/SparseLinearLayer.cs @@ -101,21 +101,26 @@ public partial class SparseLinearLayer : LayerBase _weights.NonZeroCount + OutputFeatures; /// - /// Gets whether this layer supports training. Returns false because this layer's - /// sparse weight tensor is not exposed to the tape-based autodiff path: the layer has no - /// [TrainableParameter] fields and never calls RegisterTrainableParameter, - /// so TapeTrainingStep.CollectParameters finds zero parameters here. In tape mode - /// (the default), the layer would be silently skipped while its peers train — producing - /// a partially-trainable network that is almost never the intended behavior. + /// Gets whether this layer supports training. Returns true: the layer + /// owns a working that updates its sparse + /// weight tensor and dense bias vector from gradients computed in + /// , and the legacy training path + /// (if (layer.SupportsTraining) layer.UpdateParameters(lr)) trains + /// the layer correctly. Biases are also registered as a tape trainable + /// parameter, so tape-mode optimizers update them too. /// /// - /// Sparse weight tensors don't fit the dense ParameterBuffer<T> view - /// contract used by the tape, so returning false here keeps the layer's inference - /// behavior correct while explicitly advertising that no gradient updates flow through - /// it. Integrating sparse parameters into the tape (e.g., via a sparse trainable tensor - /// view) is the follow-up required before this can flip back to true. + /// Tape-mode caveat: the sparse + /// weight tensor is still not visible to the tape's + /// ParameterBuffer<T>-based discovery — that contract requires + /// dense storage. In tape mode, weight updates flow through the + /// layer's own + + /// when the optimizer falls back to the legacy update path; weight + /// gradients do not appear in the tape's flat-buffer view. Closing that + /// gap fully (sparse-aware ParameterBuffer<T>) is tracked as + /// a follow-up. /// - public override bool SupportsTraining => false; + public override bool SupportsTraining => true; /// /// Initializes a new instance of the SparseLinearLayer. diff --git a/src/NeuralNetworks/MemoryNetwork.cs b/src/NeuralNetworks/MemoryNetwork.cs index 111ad8e86d..fce6d568a7 100644 --- a/src/NeuralNetworks/MemoryNetwork.cs +++ b/src/NeuralNetworks/MemoryNetwork.cs @@ -187,6 +187,21 @@ public MemoryNetwork(NeuralNetworkArchitecture architecture, int memorySize, _embeddingSize = embeddingSize; _memory = new Matrix(_memorySize, _embeddingSize); + // Seed memory with small random values so MemoryReadLayer has something + // non-trivial to attend over on the first forward pass. Leaving memory + // as zeros makes keys · memory^T identically zero, so softmax produces + // a uniform attention distribution and readValues = attention · memory + // is also zero — every subsequent layer then sees the same constant + // input and the network cannot discriminate between its own inputs, + // which was the source of the scaledinput_shouldchangeoutput and + // differentinputs_shouldproducedifferentoutputs failures. Use a small + // Xavier-scale so downstream layers see a reasonable magnitude. + var memRand = AiDotNet.Tensors.Helpers.RandomHelper.CreateSecureRandom(); + double memScale = Math.Sqrt(2.0 / (_memorySize + _embeddingSize)); + for (int r = 0; r < _memorySize; r++) + for (int c = 0; c < _embeddingSize; c++) + _memory[r, c] = NumOps.FromDouble((memRand.NextDouble() * 2 - 1) * memScale); + InitializeLayers(); } @@ -308,6 +323,24 @@ public override Tensor Predict(Tensor input) // Set to inference mode SetTrainingMode(false); + return RunLayers(input); + } + + /// + public override Tensor ForwardForTraining(Tensor input) + { + // The base class version walks Layers with the plain single-arg + // Forward, which misses the Memory{Read,Write}Layer memory-tensor + // plumbing and skips the rank-1 → rank-2 reshape — both of which + // Predict does. Without this override the tape-based training path + // either crashes ("TensorMatMul requires tensors of rank >= 2") + // or silently reads from an identity-memory fallback that has + // nothing to do with _memory. + return RunLayers(input); + } + + private Tensor RunLayers(Tensor input) + { // Ensure 2D input for memory operations (TensorMatMul requires rank >= 2) if (input.Rank == 1) input = input.Reshape([1, input.Shape[0]]); diff --git a/src/NeuralNetworks/MobileNetV2Network.cs b/src/NeuralNetworks/MobileNetV2Network.cs index 8a210fb3b8..46569a0206 100644 --- a/src/NeuralNetworks/MobileNetV2Network.cs +++ b/src/NeuralNetworks/MobileNetV2Network.cs @@ -239,15 +239,55 @@ public Tensor Forward(Tensor input) if (TryForwardGpuOptimized(input, out var gpuResult)) return gpuResult; - + // Accept 3D [C, H, W] (single unbatched sample) or 4D [B, C, H, W]. + // Internal layers — especially BatchNormalizationLayer — + // broadcast their channel-scale/shift tensors as [1, C, 1, 1] and + // require the input to be 4D for that broadcast to apply along the + // channel dimension. A 3D input would otherwise line up the 16-channel + // scale tensor against the spatial H=32 dimension and throw + // "Tensors with shapes [16, 32, 32] and [1, 16, 1] cannot be + // broadcast: dimension 1 has sizes 32 and 16 (must be equal or + // one must be 1)." + // inside BatchNormalizationLayer.ApplyInferenceAnyRank. The fix + // mirrors ResNet/VGG's input-shape contract. + bool addedBatch = false; Tensor output = input; + if (input.Rank == 3) + { + output = PromoteToBatchedTensor(input); + addedBatch = true; + } + foreach (var layer in Layers) { output = layer.Forward(output); } + + // If the caller passed a 3D input we expanded to 4D; squeeze the + // leading batch dim off so the output shape matches the input's + // un-batched contract. + if (addedBatch && output.Rank >= 1 && output.Shape[0] == 1) + { + int[] squeezed = new int[output.Rank - 1]; + for (int i = 0; i < squeezed.Length; i++) squeezed[i] = output.Shape[i + 1]; + output = Engine.Reshape(output, squeezed); + } return output; } + /// + public override Dictionary> GetNamedLayerActivations(Tensor input) + { + // Mirror Forward's 3D → 4D expansion so the channel-broadcasted + // BatchNormalizationLayer inside every InvertedResidualBlock sees + // the 4D tensor it requires. Without this, callers that probe + // activations layer-by-layer with a 3D test input hit the same + // "[16,32,32] vs [1,16,1] cannot be broadcast" failure that the + // Forward override already handles. + Tensor probeInput = input.Rank == 3 ? PromoteToBatchedTensor(input) : input; + return base.GetNamedLayerActivations(probeInput); + } + /// public override Tensor Predict(Tensor input) { @@ -257,7 +297,12 @@ public override Tensor Predict(Tensor input) /// public override void Train(Tensor input, Tensor expectedOutput) { - TrainWithTape(input, expectedOutput, _optimizer); + // Mirror Forward's shape contract — expand 3D inputs (and their + // targets) to 4D so ForwardForTraining's raw layer iteration + // receives the 4D tensor BatchNormalizationLayer needs. See + // ResNet/VGG Train() for the same pattern. + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); + TrainWithTape(processedInput, processedTarget, _optimizer); } /// diff --git a/src/NeuralNetworks/NeuralNetworkBase.cs b/src/NeuralNetworks/NeuralNetworkBase.cs index c8ddaacc73..791c2ecd90 100644 --- a/src/NeuralNetworks/NeuralNetworkBase.cs +++ b/src/NeuralNetworks/NeuralNetworkBase.cs @@ -2659,6 +2659,45 @@ public virtual void Train(Tensor input, Tensor expectedOutput) } } + /// + /// Promotes a rank-3 [C,H,W] tensor to rank-4 [1,C,H,W]. Named + /// PromoteToBatchedTensor to avoid collision with per-subclass + /// AddBatchDimension helpers that predate this shared utility. + /// + protected static Tensor PromoteToBatchedTensor(Tensor tensor) + { + int[] inputShape = tensor._shape; + int[] resultShape = new int[inputShape.Length + 1]; + resultShape[0] = 1; + for (int i = 0; i < inputShape.Length; i++) + { + resultShape[i + 1] = inputShape[i]; + } + return tensor.Reshape(resultShape); + } + + /// + /// Normalizes a (input, target) pair for CNN training loops: when input is + /// a single rank-3 [C,H,W] sample, adds a batch dim so downstream + /// layers see [1,C,H,W]. When the caller supplied a rank-1 + /// classification label, promotes it to match the promoted input so + /// tape-based training sees consistent shapes. + /// + /// (processedInput, processedTarget) ready for TrainWithTape. + protected static (Tensor Input, Tensor Target) EnsureBatchForCnnTraining( + Tensor input, Tensor target) + { + if (input.Rank != 3) + { + return (input, target); + } + var processedInput = PromoteToBatchedTensor(input); + var processedTarget = target.Rank < processedInput.Rank - 2 + ? PromoteToBatchedTensor(target) + : target; + return (processedInput, processedTarget); + } + /// /// Performs tape-based forward/backward pass and delegates the parameter update to the /// provided optimizer via . diff --git a/src/NeuralNetworks/QuantumNeuralNetwork.cs b/src/NeuralNetworks/QuantumNeuralNetwork.cs index 9866c1bbb7..838cf1d11c 100644 --- a/src/NeuralNetworks/QuantumNeuralNetwork.cs +++ b/src/NeuralNetworks/QuantumNeuralNetwork.cs @@ -281,20 +281,26 @@ public override Tensor Predict(Tensor input) /// public override void Train(Tensor input, Tensor expectedOutput) { - // Set training mode on all layers for proper gradient storage SetTrainingMode(true); foreach (var layer in Layers) layer.SetTrainingMode(true); - // Forward pass - var prediction = Predict(input); - - // Calculate and set the loss - LastLoss = CalculateLoss(prediction, expectedOutput); - - // Tape-based training handles gradient computation - var gradients = new List>(); - UpdateQuantumParameters(gradients); + try + { + // Use the tape-based training path like other networks. The previous + // imperative implementation ran Predict() and then called the + // optimizer's UpdateParameters(Layers), which dispatched to each + // layer's UpdateParameters(learningRate) — that expects a prior + // Backward() to have populated the per-layer gradient tensors, + // but none of that happens here, so every training call failed + // with "Backward pass must be called before updating parameters." + _trainOptimizer ??= new AdamOptimizer, Tensor>(this); + TrainWithTape(input, expectedOutput, _trainOptimizer); + } + finally + { + SetTrainingMode(false); + } } /// diff --git a/src/NeuralNetworks/ResNetNetwork.cs b/src/NeuralNetworks/ResNetNetwork.cs index fc27561c76..b3e8d84e95 100644 --- a/src/NeuralNetworks/ResNetNetwork.cs +++ b/src/NeuralNetworks/ResNetNetwork.cs @@ -441,7 +441,7 @@ public Tensor Forward(Tensor input) nameof(ResNetNetwork), "forward pass"); addedBatch = true; - processedInput = AddBatchDimension(input); + processedInput = PromoteToBatchedTensor(input); } else if (input.Rank == 4) { @@ -475,20 +475,6 @@ public Tensor Forward(Tensor input) return output; } - /// - /// Adds a batch dimension to a single input tensor. - /// - private static Tensor AddBatchDimension(Tensor input) - { - int[] inputShape = input._shape; - int[] resultShape = new int[inputShape.Length + 1]; - resultShape[0] = 1; - for (int i = 0; i < inputShape.Length; i++) - { - resultShape[i + 1] = inputShape[i]; - } - return input.Reshape(resultShape); - } /// /// Updates the parameters of all layers in the network. @@ -521,12 +507,26 @@ public override Tensor Predict(Tensor input) /// /// The input tensor for training. /// The expected output tensor (one-hot encoded class labels). + /// + /// Accepts both 3D [C, H, W] (single unbatched example) and 4D + /// [B, C, H, W] inputs — matches 's contract. + /// When a 3D input arrives, a leading batch dimension is added so every + /// downstream layer sees the conv-standard 4D tensor. The corresponding + /// target is also expanded to [1, numClasses] so the loss sees + /// matching batch dims on both operands. Previously + /// fed the raw 3D + /// tensor straight to the layer stack, which caused the FlattenLayer to + /// treat the 512-channel dimension as a batch and produce a + /// [512, 10] prediction — failing the loss shape check in + /// EnsureTargetMatchesPredicted. + /// public override void Train(Tensor input, Tensor expectedOutput) { SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { diff --git a/src/NeuralNetworks/VGGNetwork.cs b/src/NeuralNetworks/VGGNetwork.cs index 74d41f06a3..f24eba4bb9 100644 --- a/src/NeuralNetworks/VGGNetwork.cs +++ b/src/NeuralNetworks/VGGNetwork.cs @@ -271,7 +271,7 @@ public Tensor Forward(Tensor input) nameof(VGGNetwork), "forward pass"); addedBatch = true; - processedInput = AddBatchDimension(input); + processedInput = PromoteToBatchedTensor(input); } else if (input.Rank == 4) { @@ -305,29 +305,6 @@ public Tensor Forward(Tensor input) return output; } - /// - /// Adds a batch dimension to a single input tensor. - /// - /// The input tensor with shape [channels, height, width]. - /// A tensor with shape [1, channels, height, width]. - private static Tensor AddBatchDimension(Tensor input) - { - int[] inputShape = input._shape; - int[] resultShape = new int[inputShape.Length + 1]; - - // Add batch dimension of size 1 - resultShape[0] = 1; - - // Copy remaining dimensions - for (int i = 0; i < inputShape.Length; i++) - { - resultShape[i + 1] = inputShape[i]; - } - - // Use Reshape which should handle the memory layout correctly - return input.Reshape(resultShape); - } - /// /// Updates the parameters of all layers in the network. /// @@ -388,7 +365,15 @@ public override void Train(Tensor input, Tensor expectedOutput) SetTrainingMode(true); try { - TrainWithTape(input, expectedOutput, _optimizer); + // Match Forward's behavior: accept 3D [C,H,W] single example and + // expand to 4D [1,C,H,W] so the layer stack (conv / pool / flatten / + // dense) receives the batched shape it expects. Without the + // expansion the FlattenLayer in the classifier head treats the + // channel dimension as a batch and produces an output with the + // wrong leading dim — fails the loss-shape check on every training + // call with a single 3D input. + var (processedInput, processedTarget) = EnsureBatchForCnnTraining(input, expectedOutput); + TrainWithTape(processedInput, processedTarget, _optimizer); } finally { diff --git a/src/Optimizers/AdamOptimizer.cs b/src/Optimizers/AdamOptimizer.cs index cbc0c97248..96687a522d 100644 --- a/src/Optimizers/AdamOptimizer.cs +++ b/src/Optimizers/AdamOptimizer.cs @@ -482,13 +482,21 @@ public override void Step(TapeStepContext context) if (!context.Gradients.TryGetValue(param, out var grad)) continue; - // Lazily initialize per-parameter moment tensors - if (!_tapeM.TryGetValue(param, out var m)) + // Lazily initialize per-parameter moment tensors. If the parameter + // was first seen while a lazy-init layer (e.g. + // MultiHeadAttentionLayer with IsLazy: true initialization + // strategy) still had its weights allocated as a placeholder + // [0, 0] tensor, our cached m / v captured the placeholder shape. + // Once the layer materializes real weights, the gradient arrives + // at the real shape — m / v need to be re-allocated to match, + // otherwise TensorAdd's result has a length larger than m and + // TensorCopy throws "Destination array was not long enough". + if (!_tapeM.TryGetValue(param, out var m) || !m._shape.SequenceEqual(param._shape)) { m = new Tensor(param._shape); _tapeM[param] = m; } - if (!_tapeV.TryGetValue(param, out var v)) + if (!_tapeV.TryGetValue(param, out var v) || !v._shape.SequenceEqual(param._shape)) { v = new Tensor(param._shape); _tapeV[param] = v; diff --git a/src/Regression/PoissonRegression.cs b/src/Regression/PoissonRegression.cs index 1b52c47e9a..a317deaf49 100644 --- a/src/Regression/PoissonRegression.cs +++ b/src/Regression/PoissonRegression.cs @@ -181,6 +181,27 @@ public override void Train(Matrix x, Vector y) newCoefficients = Regularization.Regularize(newCoefficients); } + // Guard against NaN/Inf leaking out of the IRLS step. Even with + // ridge diagonal loading the qr/svd solve can produce non-finite + // entries when the reweighted x^t·w·x is numerically singular + // (e.g. when exp-clamping collapses most mu's to the floor), + // and propagating those poisons every subsequent predict() with + // nan. when we hit that, stop iterating with the last known-good + // coefficients rather than overwriting them — this mirrors how + // statsmodels' glm halts on a "linear algebra error". + bool hasBadEntry = false; + for (int i = 0; i < newCoefficients.Length; i++) + { + double v = NumOps.ToDouble(newCoefficients[i]); + if (double.IsNaN(v) || double.IsInfinity(v)) + { + hasBadEntry = true; + break; + } + } + if (hasBadEntry) + break; + if (HasConverged(currentCoefficients, newCoefficients)) { break; diff --git a/src/Regression/RadialBasisFunctionRegression.cs b/src/Regression/RadialBasisFunctionRegression.cs index 3d3d645c0c..7c35147043 100644 --- a/src/Regression/RadialBasisFunctionRegression.cs +++ b/src/Regression/RadialBasisFunctionRegression.cs @@ -488,30 +488,61 @@ private T RbfKernel(T distance) /// private Vector SolveLinearRegression(Matrix x, Vector y) { - // Use pseudo-inverse with ridge regularization to solve for weights - // Ridge regularization (Tikhonov regularization) adds a small penalty term (λI) - // to prevent numerical instability and improve generalization - Matrix xTranspose = x.Transpose(); - Matrix xTx = xTranspose.Multiply(x); - - // Add ridge regularization: (X^T X + λI)^-1 X^T y - // Scale lambda relative to the diagonal magnitude to ensure numerical stability - double diagMean = 0; - for (int i = 0; i < xTx.Rows; i++) - diagMean += Math.Abs(NumOps.ToDouble(xTx[i, i])); - diagMean /= xTx.Rows; - T stabilityLambda = NumOps.FromDouble( - Math.Max(MinimumStabilityLambda, diagMean * StabilityLambdaScale)); - Matrix identity = Matrix.CreateIdentity(xTx.Rows); - Matrix xTxRegularized = xTx.Add(identity.Multiply(stabilityLambda)); + // RBF design matrices are often severely ill-conditioned: when a few + // centers end up far from every input, the corresponding columns + // become near-zero and X^T·X has a huge condition number. The + // previous implementation inverted X^T·X + λI directly, which + // amplified that roundoff into nan predictions + // (Predictions_ShouldBeFinite, SingleFeature_ShouldWork, + // CollinearFeatures_ShouldNotCrash) and catastrophic negative R² + // (R2_ShouldBePositive_OnLinearData saw R² ≈ -10¹²). + // + // Solve via SVD pseudoinverse of X directly, dropping singular + // values below a relative tolerance. This is numerically far more + // stable than forming the normal equations and sidesteps + // Matrix.Inverse entirely. + var svd = new AiDotNet.DecompositionMethods.MatrixDecomposition.SvdDecomposition(x); + + T sigmaMax = NumOps.Zero; + for (int i = 0; i < svd.S.Length; i++) + { + if (NumOps.GreaterThan(svd.S[i], sigmaMax)) + sigmaMax = svd.S[i]; + } + + // Tikhonov-regularized SVD solve: weights = V · diag(σ / (σ² + λ²)) · Uᵀ · y. + // Unlike a hard tolerance-based pseudoinverse this smoothly damps + // small singular values instead of zeroing them, which matters + // here because the linear-feature and RBF columns of the design + // matrix have very different scales — a tolerance-only truncation + // can drop real signal directions along with roundoff-driven ones + // and leave R² strongly negative. Small λ ≈ 1e-6 · σ_max gives a + // stable solve while barely biasing the well-conditioned directions. + T lambda = NumOps.Multiply(sigmaMax, NumOps.FromDouble(1e-6)); + T lambdaSq = NumOps.Multiply(lambda, lambda); + + var weights = new Vector(svd.Vt.Columns); + for (int i = 0; i < svd.S.Length; i++) + { + T sigma = svd.S[i]; + T sigmaSq = NumOps.Multiply(sigma, sigma); + T damped = NumOps.Divide(sigma, NumOps.Add(sigmaSq, lambdaSq)); + + Vector uCol = svd.U.GetColumn(i); + T coeff = NumOps.Multiply(uCol.DotProduct(y), damped); + Vector vtRow = svd.Vt.GetRow(i); + weights = weights.Add(vtRow.Multiply(coeff)); + } + + // Apply external regularization (if any) on the learned weight + // vector — the pseudoinverse path doesn't build the normal-equations + // matrix that the previous code was regularizing. if (Regularization != null) { - xTxRegularized = xTxRegularized.Add(Regularization.Regularize(xTx)); + weights = Regularization.Regularize(weights); } - Matrix xTxInverse = xTxRegularized.Inverse(); - Matrix xTxInverseXT = xTxInverse.Multiply(xTranspose); - return xTxInverseXT.Multiply(y); + return weights; } /// diff --git a/src/Statistics/BasicStats.cs b/src/Statistics/BasicStats.cs index 2d52203bd4..1fbc2763cc 100644 --- a/src/Statistics/BasicStats.cs +++ b/src/Statistics/BasicStats.cs @@ -436,18 +436,40 @@ public static BasicStats Empty() /// private void CalculateStats(Vector values) { - N = values.Length; - if (N == 0) return; - Mean = values.Average(); - Variance = values.Variance(); - StandardDeviation = _numOps.Sqrt(Variance); - (Skewness, Kurtosis) = StatisticsHelper.CalculateSkewnessAndKurtosis(values, Mean, StandardDeviation, N); - Min = values.Min(); - Max = values.Max(); - Median = StatisticsHelper.CalculateMedian(values); - (FirstQuartile, ThirdQuartile) = StatisticsHelper.CalculateQuantiles(values); - InterquartileRange = _numOps.Subtract(ThirdQuartile, FirstQuartile); - MAD = StatisticsHelper.CalculateMeanAbsoluteDeviation(values, Median); + // CRITICAL: every property on this class (N, Mean, Variance, + // StandardDeviation, Median, etc.) goes through a getter that calls + // EnsureFullStatsComputed(), which calls back into this method + // when _fullStatsComputed is false. We are exactly in that window + // here, so reading any property would recurse forever and crash + // the test host with a StackOverflowException. Compute everything + // into locals and assign to the properties at the end. + int n = values.Length; + N = n; + if (n == 0) return; + + T mean = values.Average(); + T variance = values.Variance(); + T stdDev = _numOps.Sqrt(variance); + var (skewness, kurtosis) = StatisticsHelper.CalculateSkewnessAndKurtosis(values, mean, stdDev, n); + T min = values.Min(); + T max = values.Max(); + T median = StatisticsHelper.CalculateMedian(values); + var (firstQuartile, thirdQuartile) = StatisticsHelper.CalculateQuantiles(values); + T iqr = _numOps.Subtract(thirdQuartile, firstQuartile); + T mad = StatisticsHelper.CalculateMeanAbsoluteDeviation(values, median); + + Mean = mean; + Variance = variance; + StandardDeviation = stdDev; + Skewness = skewness; + Kurtosis = kurtosis; + Min = min; + Max = max; + Median = median; + FirstQuartile = firstQuartile; + ThirdQuartile = thirdQuartile; + InterquartileRange = iqr; + MAD = mad; } /// diff --git a/src/Statistics/ErrorStats.cs b/src/Statistics/ErrorStats.cs index 7e72d7f123..f02a78c636 100644 --- a/src/Statistics/ErrorStats.cs +++ b/src/Statistics/ErrorStats.cs @@ -513,50 +513,83 @@ public static ErrorStats Empty() /// private void CalculateErrorStats(Vector actual, Vector predicted, int numberOfParameters, PredictionType predictionType = PredictionType.Regression) { + // All intermediate values are held in locals and only assigned to the + // observable properties at the very end. Earlier code called + // RMSE = _numOps.Sqrt(MSE); + // AIC = StatisticsHelper.CalculateAIC(n, numberOfParameters, RSS); + // which read MSE/RSS through their property getters. Those getters call + // EnsureFullStatsComputed, which is still running CalculateErrorStats — + // so the property read re-enters CalculateErrorStats and recurses + // without bound. The test host crashes with StackOverflowException + // (Classification / Clustering / Regression / TimeSeries / Serving + // host-crashes on PR #1154 and master). int n = actual.Length; - // Calculate basic error metrics - MAE = StatisticsHelper.CalculateMeanAbsoluteError(actual, predicted); - RSS = StatisticsHelper.CalculateResidualSumOfSquares(actual, predicted); - MSE = StatisticsHelper.CalculateMeanSquaredError(actual, predicted); - RMSE = _numOps.Sqrt(MSE); - MAPE = StatisticsHelper.CalculateMeanAbsolutePercentageError(actual, predicted); - MedianAbsoluteError = StatisticsHelper.CalculateMedianAbsoluteError(actual, predicted); - MaxError = StatisticsHelper.CalculateMaxError(actual, predicted); + T mae = StatisticsHelper.CalculateMeanAbsoluteError(actual, predicted); + T rss = StatisticsHelper.CalculateResidualSumOfSquares(actual, predicted); + T mse = StatisticsHelper.CalculateMeanSquaredError(actual, predicted); + T rmse = _numOps.Sqrt(mse); + T mape = StatisticsHelper.CalculateMeanAbsolutePercentageError(actual, predicted); + T medAbs = StatisticsHelper.CalculateMedianAbsoluteError(actual, predicted); + T maxErr = StatisticsHelper.CalculateMaxError(actual, predicted); + T aucPR, aucROC; if (predictionType == PredictionType.BinaryClassification) { - AUCPR = StatisticsHelper.CalculatePrecisionRecallAUC(actual, predicted); - AUCROC = StatisticsHelper.CalculateROCAUC(actual, predicted); + aucPR = StatisticsHelper.CalculatePrecisionRecallAUC(actual, predicted); + aucROC = StatisticsHelper.CalculateROCAUC(actual, predicted); } else { - AUCPR = _numOps.Zero; - AUCROC = _numOps.Zero; + aucPR = _numOps.Zero; + aucROC = _numOps.Zero; } - SMAPE = StatisticsHelper.CalculateSymmetricMeanAbsolutePercentageError(actual, predicted); - MeanSquaredLogError = StatisticsHelper.CalculateMeanSquaredLogError(actual, predicted); - CRPS = StatisticsHelper.CalculateCRPS(actual, predicted); - - // Calculate standard errors - SampleStandardError = StatisticsHelper.CalculateSampleStandardError(actual, predicted, numberOfParameters); - PopulationStandardError = StatisticsHelper.CalculatePopulationStandardError(actual, predicted); - - // Calculate bias and autocorrelation metrics - MeanBiasError = StatisticsHelper.CalculateMeanBiasError(actual, predicted); - TheilUStatistic = StatisticsHelper.CalculateTheilUStatistic(actual, predicted); - DurbinWatsonStatistic = StatisticsHelper.CalculateDurbinWatsonStatistic(actual, predicted); - - // Calculate information criteria - AIC = StatisticsHelper.CalculateAIC(n, numberOfParameters, RSS); - BIC = StatisticsHelper.CalculateBIC(n, numberOfParameters, RSS); - AICAlt = StatisticsHelper.CalculateAICAlternative(n, numberOfParameters, RSS); - - // Calculate classification metrics - Accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); - (Precision, Recall, F1Score) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); - - // Populate error list - ErrorList = new List(StatisticsHelper.CalculateResiduals(actual, predicted)); + T smape = StatisticsHelper.CalculateSymmetricMeanAbsolutePercentageError(actual, predicted); + T msle = StatisticsHelper.CalculateMeanSquaredLogError(actual, predicted); + T crps = StatisticsHelper.CalculateCRPS(actual, predicted); + + T sampleStd = StatisticsHelper.CalculateSampleStandardError(actual, predicted, numberOfParameters); + T popStd = StatisticsHelper.CalculatePopulationStandardError(actual, predicted); + + T bias = StatisticsHelper.CalculateMeanBiasError(actual, predicted); + T theil = StatisticsHelper.CalculateTheilUStatistic(actual, predicted); + T dw = StatisticsHelper.CalculateDurbinWatsonStatistic(actual, predicted); + + T aic = StatisticsHelper.CalculateAIC(n, numberOfParameters, rss); + T bic = StatisticsHelper.CalculateBIC(n, numberOfParameters, rss); + T aicAlt = StatisticsHelper.CalculateAICAlternative(n, numberOfParameters, rss); + + T accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); + var (precision, recall, f1) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + + var residuals = new List(StatisticsHelper.CalculateResiduals(actual, predicted)); + + // Assign to the observable properties only after every dependency is a + // local — no re-entry possible now. + MAE = mae; + RSS = rss; + MSE = mse; + RMSE = rmse; + MAPE = mape; + MedianAbsoluteError = medAbs; + MaxError = maxErr; + AUCPR = aucPR; + AUCROC = aucROC; + SMAPE = smape; + MeanSquaredLogError = msle; + CRPS = crps; + SampleStandardError = sampleStd; + PopulationStandardError = popStd; + MeanBiasError = bias; + TheilUStatistic = theil; + DurbinWatsonStatistic = dw; + AIC = aic; + BIC = bic; + AICAlt = aicAlt; + Accuracy = accuracy; + Precision = precision; + Recall = recall; + F1Score = f1; + ErrorList = residuals; } /// diff --git a/src/Statistics/ModelStats.cs b/src/Statistics/ModelStats.cs index 23fd1ef8b5..0b9d8f3552 100644 --- a/src/Statistics/ModelStats.cs +++ b/src/Statistics/ModelStats.cs @@ -552,14 +552,16 @@ public static ModelStats Empty() /// private void CalculateModelStats(ModelStatsInputs inputs) { - // Convert input matrix to Matrix for statistical calculations + // All intermediate values held in locals until the end. Previous code + // read CorrelationMatrix and CovarianceMatrix through their property + // getters during computation; those getters call + // EnsureFullStatsComputed, which re-enters this method — unbounded + // recursion + StackOverflowException. Same bug class as BasicStats / + // ErrorStats / PredictionStats. Matrix matrix = ConversionsHelper.ConvertToMatrix(inputs.XMatrix); - - // Convert actual and predicted values to Vector for statistical calculations Vector actual = ConversionsHelper.ConvertToVector(inputs.Actual); Vector predicted = ConversionsHelper.ConvertToVector(inputs.Predicted); - // Convert coefficients if available Vector coefficients = Vector.Empty(); if (inputs.Coefficients != null) { @@ -568,64 +570,89 @@ private void CalculateModelStats(ModelStatsInputs inputs) var featureCount = inputs.FeatureCount; - // Calculate all statistical metrics using the converted data types - CorrelationMatrix = StatisticsHelper.CalculateCorrelationMatrix(matrix, _options); - CovarianceMatrix = StatisticsHelper.CalculateCovarianceMatrix(matrix); - VIFList = StatisticsHelper.CalculateVIF(CorrelationMatrix, _options); - ConditionNumber = StatisticsHelper.CalculateConditionNumber(matrix, _options); - LogPointwisePredictiveDensity = StatisticsHelper.CalculateLogPointwisePredictiveDensity(actual, predicted); + var correlationMatrix = StatisticsHelper.CalculateCorrelationMatrix(matrix, _options); + var covarianceMatrix = StatisticsHelper.CalculateCovarianceMatrix(matrix); + var vifList = StatisticsHelper.CalculateVIF(correlationMatrix, _options); + T conditionNumber = StatisticsHelper.CalculateConditionNumber(matrix, _options); + T logPpd = StatisticsHelper.CalculateLogPointwisePredictiveDensity(actual, predicted); + List? looPd = null; if (inputs.FitFunction != null) { - // Convert the fit function to work with Matrix and Vector var convertedFitFunction = ConversionsHelper.ConvertFitFunction(inputs.FitFunction); - - // Create a wrapper function that matches the expected signature Vector wrappedFitFunction(Matrix m, Vector v) => convertedFitFunction(m); - LeaveOneOutPredictiveDensities = StatisticsHelper.CalculateLeaveOneOutPredictiveDensities(matrix, actual, wrappedFitFunction); + looPd = StatisticsHelper.CalculateLeaveOneOutPredictiveDensities(matrix, actual, wrappedFitFunction); } - ObservedTestStatistic = StatisticsHelper.CalculateObservedTestStatistic(actual, predicted); - PosteriorPredictiveSamples = StatisticsHelper.CalculatePosteriorPredictiveSamples(actual, predicted, featureCount); - MarginalLikelihood = StatisticsHelper.CalculateMarginalLikelihood(actual, predicted, featureCount); - ReferenceModelMarginalLikelihood = StatisticsHelper.CalculateReferenceModelMarginalLikelihood(actual); - LogLikelihood = StatisticsHelper.CalculateLogLikelihood(actual, predicted); - EffectiveNumberOfParameters = StatisticsHelper.CalculateEffectiveNumberOfParameters(matrix, coefficients); - MutualInformation = StatisticsHelper.CalculateMutualInformation(actual, predicted); - NormalizedMutualInformation = StatisticsHelper.CalculateNormalizedMutualInformation(actual, predicted); - VariationOfInformation = StatisticsHelper.CalculateVariationOfInformation(actual, predicted); - SilhouetteScore = StatisticsHelper.CalculateSilhouetteScore(matrix, predicted); - CalinskiHarabaszIndex = StatisticsHelper.CalculateCalinskiHarabaszIndex(matrix, predicted); - DaviesBouldinIndex = StatisticsHelper.CalculateDaviesBouldinIndex(matrix, predicted); - MeanAveragePrecision = StatisticsHelper.CalculateMeanAveragePrecision(actual, predicted, _options.MapTopK); - NormalizedDiscountedCumulativeGain = StatisticsHelper.CalculateNDCG(actual, predicted, _options.NdcgTopK); - MeanReciprocalRank = StatisticsHelper.CalculateMeanReciprocalRank(actual, predicted); - - // Calculate residuals and time series metrics + T observedTest = StatisticsHelper.CalculateObservedTestStatistic(actual, predicted); + var posteriorSamples = StatisticsHelper.CalculatePosteriorPredictiveSamples(actual, predicted, featureCount); + T marginalLik = StatisticsHelper.CalculateMarginalLikelihood(actual, predicted, featureCount); + T refModelML = StatisticsHelper.CalculateReferenceModelMarginalLikelihood(actual); + T logLik = StatisticsHelper.CalculateLogLikelihood(actual, predicted); + T effParams = StatisticsHelper.CalculateEffectiveNumberOfParameters(matrix, coefficients); + T mi = StatisticsHelper.CalculateMutualInformation(actual, predicted); + T nmi = StatisticsHelper.CalculateNormalizedMutualInformation(actual, predicted); + T voi = StatisticsHelper.CalculateVariationOfInformation(actual, predicted); + T silhouette = StatisticsHelper.CalculateSilhouetteScore(matrix, predicted); + T ch = StatisticsHelper.CalculateCalinskiHarabaszIndex(matrix, predicted); + T db = StatisticsHelper.CalculateDaviesBouldinIndex(matrix, predicted); + T mAP = StatisticsHelper.CalculateMeanAveragePrecision(actual, predicted, _options.MapTopK); + T ndcg = StatisticsHelper.CalculateNDCG(actual, predicted, _options.NdcgTopK); + T mrr = StatisticsHelper.CalculateMeanReciprocalRank(actual, predicted); + var residuals = StatisticsHelper.CalculateResiduals(actual, predicted); - AutoCorrelationFunction = StatisticsHelper.CalculateAutoCorrelationFunction(residuals, _options.AcfMaxLag); - PartialAutoCorrelationFunction = StatisticsHelper.CalculatePartialAutoCorrelationFunction(residuals, _options.PacfMaxLag); - - // Calculate distance metrics - EuclideanDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Euclidean); - ManhattanDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Manhattan); - CosineSimilarity = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Cosine); - JaccardSimilarity = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Jaccard); - HammingDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Hamming); - - // Mahalanobis distance requires vector dimensions to match covariance matrix dimensions - // Skip calculation if dimensions don't match (covariance is feature x feature, not sample x sample) + var acf = StatisticsHelper.CalculateAutoCorrelationFunction(residuals, _options.AcfMaxLag); + var pacf = StatisticsHelper.CalculatePartialAutoCorrelationFunction(residuals, _options.PacfMaxLag); + + T euclidean = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Euclidean); + T manhattan = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Manhattan); + T cosine = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Cosine); + T jaccard = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Jaccard); + T hamming = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Hamming); + + T mahalanobis = _numOps.Zero; try { - if (CovarianceMatrix.Rows == actual.Length && CovarianceMatrix.Columns == actual.Length) + if (covarianceMatrix.Rows == actual.Length && covarianceMatrix.Columns == actual.Length) { - MahalanobisDistance = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Mahalanobis, CovarianceMatrix); + mahalanobis = StatisticsHelper.CalculateDistance(actual, predicted, DistanceMetricType.Mahalanobis, covarianceMatrix); } } catch (ArgumentException) { - // Silently skip Mahalanobis distance calculation when dimensions don't match + // Silently skip Mahalanobis distance calculation when dimensions don't match. } + + // Assign properties once every dependency is a local — no re-entry. + CorrelationMatrix = correlationMatrix; + CovarianceMatrix = covarianceMatrix; + VIFList = vifList; + ConditionNumber = conditionNumber; + LogPointwisePredictiveDensity = logPpd; + if (looPd != null) LeaveOneOutPredictiveDensities = looPd; + ObservedTestStatistic = observedTest; + PosteriorPredictiveSamples = posteriorSamples; + MarginalLikelihood = marginalLik; + ReferenceModelMarginalLikelihood = refModelML; + LogLikelihood = logLik; + EffectiveNumberOfParameters = effParams; + MutualInformation = mi; + NormalizedMutualInformation = nmi; + VariationOfInformation = voi; + SilhouetteScore = silhouette; + CalinskiHarabaszIndex = ch; + DaviesBouldinIndex = db; + MeanAveragePrecision = mAP; + NormalizedDiscountedCumulativeGain = ndcg; + MeanReciprocalRank = mrr; + AutoCorrelationFunction = acf; + PartialAutoCorrelationFunction = pacf; + EuclideanDistance = euclidean; + ManhattanDistance = manhattan; + CosineSimilarity = cosine; + JaccardSimilarity = jaccard; + HammingDistance = hamming; + MahalanobisDistance = mahalanobis; } /// diff --git a/src/Statistics/PredictionStats.cs b/src/Statistics/PredictionStats.cs index 65247b21ab..31384121d2 100644 --- a/src/Statistics/PredictionStats.cs +++ b/src/Statistics/PredictionStats.cs @@ -663,34 +663,72 @@ public static PredictionStats WithR2Only(T r2) /// private void CalculatePredictionStats(Vector actual, Vector predicted, int numberOfParameters, T confidenceLevel, int learningCurveSteps, PredictionType predictionType) { - BestDistributionFit = StatisticsHelper.DetermineBestFitDistribution(predicted); - - MeanPredictionError = StatisticsHelper.CalculateMeanPredictionError(actual, predicted); - MedianPredictionError = StatisticsHelper.CalculateMedianPredictionError(actual, predicted); - - R2 = StatisticsHelper.CalculateR2(actual, predicted); - AdjustedR2 = StatisticsHelper.CalculateAdjustedR2(R2, actual.Length, numberOfParameters); - ExplainedVarianceScore = StatisticsHelper.CalculateExplainedVarianceScore(actual, predicted); - LearningCurve = StatisticsHelper.CalculateLearningCurve(actual, predicted, learningCurveSteps); - PearsonCorrelation = StatisticsHelper.CalculatePearsonCorrelationCoefficient(actual, predicted); - SpearmanCorrelation = StatisticsHelper.CalculateSpearmanRankCorrelationCoefficient(actual, predicted); - KendallTau = StatisticsHelper.CalculateKendallTau(actual, predicted); - DynamicTimeWarping = StatisticsHelper.CalculateDynamicTimeWarping(actual, predicted); - - PredictionInterval = StatisticsHelper.CalculatePredictionIntervals(actual, predicted, confidenceLevel); - PredictionIntervalCoverage = StatisticsHelper.CalculatePredictionIntervalCoverage(actual, predicted, PredictionInterval.Lower, PredictionInterval.Upper); - ConfidenceInterval = StatisticsHelper.CalculateConfidenceIntervals(predicted, confidenceLevel, BestDistributionFit.DistributionType); - CredibleInterval = StatisticsHelper.CalculateCredibleIntervals(predicted, confidenceLevel, BestDistributionFit.DistributionType); - ToleranceInterval = StatisticsHelper.CalculateToleranceInterval(actual, predicted, confidenceLevel); - ForecastInterval = StatisticsHelper.CalculateForecastInterval(actual, predicted, confidenceLevel); - QuantileIntervals = StatisticsHelper.CalculateQuantileIntervals(actual, predicted, new T[] { _numOps.FromDouble(0.25), _numOps.FromDouble(0.5), _numOps.FromDouble(0.75) }); - BootstrapInterval = StatisticsHelper.CalculateBootstrapInterval(actual, predicted, confidenceLevel); - SimultaneousPredictionInterval = StatisticsHelper.CalculateSimultaneousPredictionInterval(actual, predicted, confidenceLevel); - JackknifeInterval = StatisticsHelper.CalculateJackknifeInterval(actual, predicted); - PercentileInterval = StatisticsHelper.CalculatePercentileInterval(predicted, confidenceLevel); - - Accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); - (Precision, Recall, F1Score) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + // All intermediate values computed into locals; properties are set + // together at the bottom. Before this rewrite the method read + // AdjustedR2's dependency via R2's property getter, which calls + // EnsureFullStatsComputed and re-enters this method — unbounded + // recursion + StackOverflowException in the test host. Same class + // of bug as BasicStats / ErrorStats; see those files for context. + var bestDist = StatisticsHelper.DetermineBestFitDistribution(predicted); + + T meanPredErr = StatisticsHelper.CalculateMeanPredictionError(actual, predicted); + T medianPredErr = StatisticsHelper.CalculateMedianPredictionError(actual, predicted); + + // R2 and AdjustedR2 were already computed eagerly in the ctor + // (see lines 570-571) with the same Actual/Predicted/NumberOfParameters + // inputs. Reuse the stored values instead of calling StatisticsHelper + // a second time — cuts a redundant O(n) scan per metric over the + // prediction vector on the lazy-compute path. + T r2 = R2; + T adjR2 = AdjustedR2; + T evs = StatisticsHelper.CalculateExplainedVarianceScore(actual, predicted); + var lc = StatisticsHelper.CalculateLearningCurve(actual, predicted, learningCurveSteps); + T pearson = StatisticsHelper.CalculatePearsonCorrelationCoefficient(actual, predicted); + T spearman = StatisticsHelper.CalculateSpearmanRankCorrelationCoefficient(actual, predicted); + T kendall = StatisticsHelper.CalculateKendallTau(actual, predicted); + T dtw = StatisticsHelper.CalculateDynamicTimeWarping(actual, predicted); + + (T Lower, T Upper) predInterval = StatisticsHelper.CalculatePredictionIntervals(actual, predicted, confidenceLevel); + T predIntervalCov = StatisticsHelper.CalculatePredictionIntervalCoverage(actual, predicted, predInterval.Lower, predInterval.Upper); + var confInterval = StatisticsHelper.CalculateConfidenceIntervals(predicted, confidenceLevel, bestDist.DistributionType); + var credInterval = StatisticsHelper.CalculateCredibleIntervals(predicted, confidenceLevel, bestDist.DistributionType); + var tolInterval = StatisticsHelper.CalculateToleranceInterval(actual, predicted, confidenceLevel); + var forecastInterval = StatisticsHelper.CalculateForecastInterval(actual, predicted, confidenceLevel); + var quantiles = StatisticsHelper.CalculateQuantileIntervals(actual, predicted, new T[] { _numOps.FromDouble(0.25), _numOps.FromDouble(0.5), _numOps.FromDouble(0.75) }); + var bootstrap = StatisticsHelper.CalculateBootstrapInterval(actual, predicted, confidenceLevel); + var simulPred = StatisticsHelper.CalculateSimultaneousPredictionInterval(actual, predicted, confidenceLevel); + var jackknife = StatisticsHelper.CalculateJackknifeInterval(actual, predicted); + var percentile = StatisticsHelper.CalculatePercentileInterval(predicted, confidenceLevel); + + T accuracy = StatisticsHelper.CalculateAccuracy(actual, predicted, predictionType); + var (precision, recall, f1) = StatisticsHelper.CalculatePrecisionRecallF1(actual, predicted, predictionType); + + BestDistributionFit = bestDist; + MeanPredictionError = meanPredErr; + MedianPredictionError = medianPredErr; + R2 = r2; + AdjustedR2 = adjR2; + ExplainedVarianceScore = evs; + LearningCurve = lc; + PearsonCorrelation = pearson; + SpearmanCorrelation = spearman; + KendallTau = kendall; + DynamicTimeWarping = dtw; + PredictionInterval = predInterval; + PredictionIntervalCoverage = predIntervalCov; + ConfidenceInterval = confInterval; + CredibleInterval = credInterval; + ToleranceInterval = tolInterval; + ForecastInterval = forecastInterval; + QuantileIntervals = quantiles; + BootstrapInterval = bootstrap; + SimultaneousPredictionInterval = simulPred; + JackknifeInterval = jackknife; + PercentileInterval = percentile; + Accuracy = accuracy; + Precision = precision; + Recall = recall; + F1Score = f1; } /// diff --git a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs index c120d52a7f..95bc65be92 100644 --- a/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs +++ b/tests/AiDotNet.Tests/Data/RobustFileOpsMoveRetryTests.cs @@ -46,56 +46,43 @@ public async Task Move_Succeeds_WhenNoContention() } /// - /// Simulates the Windows antivirus / indexer lock pattern: open the source - /// file with exclusive sharing for a short window, then release it. - /// The retry loop must tolerate the transient IOException and succeed - /// once the lock is released. + /// Cross-platform retry-trigger: the destination's parent directory + /// doesn't exist when the move starts, so File.Move throws + /// DirectoryNotFoundException (a subclass of IOException). A + /// background task creates the parent ~250 ms later, after which a + /// retry attempt succeeds. This exercises the same retry path as + /// the Windows AV / indexer scenario without depending on Windows- + /// specific FileShare.None semantics — on Linux, opening a + /// FileStream with FileShare.None does not actually block File.Move, + /// so the original lock-based simulation passed trivially without + /// ever exercising retry on the Linux CI runner. /// [Fact] - public async Task Move_SucceedsAfter_TransientSharingViolation() + public async Task Move_SucceedsAfter_TransientMissingParentDirectory() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); - string dst = Path.Combine(Path.GetTempPath(), $"aidotnet_move_dst_{Guid.NewGuid()}.bin"); + string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}"); + string dst = Path.Combine(parentDir, "dst.bin"); File.WriteAllBytes(src, [9, 9, 9]); - // Acquire an exclusive handle on the source, synchronize with the - // main test so the retry path is actually exercised (otherwise - // Task.Run scheduling could let the main thread call File.Move - // before the lock exists and the test passes trivially without - // ever triggering retry), hold the handle for ~250 ms, then - // release so the retry loop eventually succeeds. Retry delay is - // scaled up to 100 ms per attempt here to keep the race window - // deterministic; production default is 200 ms. - var lockAcquired = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var lockRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + // Background task creates the parent directory after ~250 ms, + // letting the retry loop succeed once the directory exists. + // Retry schedule: 100, 200, 300, 400 ms (initialDelayMs * attempt), + // so by attempt 3 or 4 the directory will be in place. + var dirReady = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _ = Task.Run(async () => { try { - using var holder = new FileStream(src, FileMode.Open, FileAccess.Read, FileShare.None); - lockAcquired.TrySetResult(true); await Task.Delay(TimeSpan.FromMilliseconds(250)); - } - catch (Exception ex) - { - // If the FileStream open itself fails, release both - // synchronization primitives so the main test fails fast - // with a clear error rather than hanging on the wait. - lockAcquired.TrySetException(ex); + Directory.CreateDirectory(parentDir); } finally { - lockRelease.TrySetResult(true); + dirReady.TrySetResult(true); } }); - // Wait for the background task to actually acquire the exclusive - // handle before we start the move. Without this await the test - // is a race between Task.Run scheduling and the main thread, and - // the retry path would not be exercised on the "fast" side of - // the race — a silently-passing test. - await lockAcquired.Task; - try { await RobustFileOps.MoveWithRetryAsync(src, dst, CancellationToken.None, maxAttempts: 8, initialDelayMs: 100); @@ -105,14 +92,12 @@ public async Task Move_SucceedsAfter_TransientSharingViolation() } finally { - // Drain the background task before teardown regardless of the - // assertion outcome. If we didn't await here and an assertion - // above threw, the background task could still own the - // FileStream when File.Delete(src) runs, masking the real - // failure with a confusing "file in use" IOException. - await lockRelease.Task; + // Drain the background task before teardown so cleanup is + // deterministic regardless of which assertion threw. + await dirReady.Task; if (File.Exists(src)) File.Delete(src); if (File.Exists(dst)) File.Delete(dst); + if (Directory.Exists(parentDir)) Directory.Delete(parentDir, recursive: true); } } @@ -175,29 +160,35 @@ public void ReplaceSync_Succeeds_WhenNoContention() /// /// If every retry attempt fails, the final attempt must propagate the /// underlying exception rather than silently succeed or hang. + /// + /// Cross-platform retry-trigger: destination's parent directory never + /// exists, so every File.Move attempt throws DirectoryNotFoundException + /// (an IOException subclass). Assert.ThrowsAsync<DirectoryNotFoundException> + /// verifies the specific cross-platform trigger used by this test. /// [Fact] - public async Task Move_Propagates_WhenLockNeverReleases() + public async Task Move_Propagates_WhenParentDirectoryNeverCreated() { string src = Path.Combine(Path.GetTempPath(), $"aidotnet_move_{Guid.NewGuid()}.bin"); - string dst = Path.Combine(Path.GetTempPath(), $"aidotnet_move_dst_{Guid.NewGuid()}.bin"); + string parentDir = Path.Combine(Path.GetTempPath(), $"aidotnet_move_parent_{Guid.NewGuid()}"); + string dst = Path.Combine(parentDir, "dst.bin"); File.WriteAllBytes(src, [1]); - // Hold an exclusive read handle for the duration of the call — all - // attempts will fail with IOException and the final one should - // surface. - using var holder = new FileStream(src, FileMode.Open, FileAccess.Read, FileShare.None); + // Parent directory is never created — every retry attempt fails, + // and the final attempt must propagate the IOException to the + // caller (rather than swallow it). try { - await Assert.ThrowsAsync( + await Assert.ThrowsAsync( () => RobustFileOps.MoveWithRetryAsync(src, dst, CancellationToken.None, maxAttempts: 3, initialDelayMs: 1)); Assert.False(File.Exists(dst), "Destination must not exist when the move fails"); + Assert.True(File.Exists(src), "Source must remain in place when the move fails"); } finally { - holder.Dispose(); if (File.Exists(src)) File.Delete(src); if (File.Exists(dst)) File.Delete(dst); + if (Directory.Exists(parentDir)) Directory.Delete(parentDir, recursive: true); } } } diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs index 41c71e3963..4b6b6d227f 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/InstructorEmbeddingTests.cs @@ -6,6 +6,16 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class InstructorEmbeddingTests : EmbeddingModelTestBase { + // InstructorEmbedding's default ctor wires a 768-dim transformer + // (inputSize=768, outputSize=768). The test base defaults to [1, 4] + // input and [1, 1] target, which caused the loss computation to try + // subtracting a [1, 768] prediction from a [1, 1] target and throw + // "Tensor shapes must match. Got [1, 768] and [1, 1]." in + // MeanSquaredErrorLoss.ComputeTapeLoss. Align the test shapes with + // the model's actual input/output dimensions. + protected override int[] InputShape => [1, 768]; + protected override int[] OutputShape => [1, 768]; + protected override INeuralNetworkModel CreateNetwork() => new InstructorEmbedding(); } diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs index 1ce24c65a8..af90d6a74f 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/MobileNetV2NetworkTests.cs @@ -1,3 +1,5 @@ +using AiDotNet.Configuration; +using AiDotNet.Enums; using AiDotNet.Interfaces; using AiDotNet.NeuralNetworks; using AiDotNet.Tests.ModelFamilyTests.Base; @@ -9,6 +11,17 @@ public class MobileNetV2NetworkTests : NeuralNetworkModelTestBase protected override int[] InputShape => [3, 64, 64]; protected override int[] OutputShape => [10]; + // The parameterless MobileNetV2Network() constructor defaults to 1000 + // ImageNet classes and 224x224 input — incompatible with this test's + // small 3x64x64 probe and 10-class OutputShape assertion. Use the + // architecture-aware overload so the network classifier head matches + // the test's expected output dimension. protected override INeuralNetworkModel CreateNetwork() - => new MobileNetV2Network(); + => new MobileNetV2Network( + new NeuralNetworkArchitecture( + inputType: InputType.ThreeDimensional, + taskType: NeuralNetworkTaskType.MultiClassClassification, + inputHeight: 64, inputWidth: 64, inputDepth: 3, + outputSize: 10), + MobileNetV2Configuration.CreateStandard(numClasses: 10)); } diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs new file mode 100644 index 0000000000..a105d4da4c --- /dev/null +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/TinyBERTNERTests.cs @@ -0,0 +1,26 @@ +using AiDotNet.Enums; +using AiDotNet.Interfaces; +using AiDotNet.NER.TransformerBased; +using AiDotNet.NeuralNetworks; +using AiDotNet.Tests.ModelFamilyTests.Base; + +namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; + +public class TinyBERTNERTests : TransformerNERTestBase +{ + // TinyBERT overrides TransformerNEROptions.HiddenDimension to 312 + // (vs the 768 default used by BERT-base). The generator's default + // [8, 768] InputShape would fail MultiHeadAttention weight matching, + // so we pin it to 312 here and construct the model via its + // architecture-only constructor (which wires up CreateTinyBERTDefaults + // with the 312-dim attention weights). + protected override int[] InputShape => [8, 312]; + + protected override INeuralNetworkModel CreateNetwork() + => new TinyBERTNER( + new NeuralNetworkArchitecture( + inputType: InputType.OneDimensional, + taskType: NeuralNetworkTaskType.Regression, + inputSize: 128, + outputSize: 4)); +} diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs index 79b80a3119..107bdd4ef1 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/UNet3DTests.cs @@ -6,9 +6,17 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class UNet3DTests : NeuralNetworkModelTestBase { - // UNet3D default: 32x32x32 voxels, 1 channel, outputSize=1 + // UNet3D is a per-voxel segmentation network: it emits one class + // prediction per input voxel, so the output carries the same spatial + // dimensions as the input. The final 1x1x1 Conv3D produces + // [numClasses, D, H, W] per sample — for the default single-class + // config that is [1, 32, 32, 32], NOT [1] (which is what the previous + // OutputShape claim produced). Without this correction the training + // tests threw "Tensor shapes must match. Got [1, 32, 32, 32] and [1]" + // when the loss tried to subtract a per-voxel prediction from a + // scalar target. protected override int[] InputShape => [1, 32, 32, 32]; - protected override int[] OutputShape => [1]; + protected override int[] OutputShape => [1, 32, 32, 32]; protected override INeuralNetworkModel CreateNetwork() => new UNet3D(); diff --git a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs index 8ba858b420..87bdddaee9 100644 --- a/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs +++ b/tests/AiDotNet.Tests/ModelFamilyTests/NeuralNetworks/Word2VecTests.cs @@ -6,6 +6,14 @@ namespace AiDotNet.Tests.ModelFamilyTests.NeuralNetworks; public class Word2VecTests : NeuralNetworkModelTestBase { + // Word2Vec's default ctor uses vocabSize=10000 — the last layer emits + // a 10000-dim softmax over the vocabulary, so predicted output length + // is 10000 per sample, not the [1, 1] implied by the base-class + // default OutputShape. Align both sides so + // OutputDimension_ShouldMatchExpectedShape compares like with like. + protected override int[] InputShape => [1, 4]; + protected override int[] OutputShape => [1, 10000]; + protected override INeuralNetworkModel CreateNetwork() => new Word2Vec(); }