feat: complete uncertainty quantification MVP

Author: ooples

src/Helpers/LayerHelper.cs

@@ -1989,6 +1989,79 @@ public static IEnumerable<ILayer<T>> CreateDefaultNeuralNetworkLayers(NeuralNetw
         yield return new ActivationLayer<T>(new[] { outputSize }, new SoftmaxActivation<T>() as IActivationFunction<T>);
     }
 
+    /// <summary>
+    /// Creates a default configuration of layers for a Bayesian neural network (Bayes-by-Backprop style).
+    /// </summary>
+    /// <remarks>
+    /// This mirrors the library's default dense+activation patterns, but uses Bayesian dense layers so the network can
+    /// express epistemic uncertainty through weight distributions.
+    /// </remarks>
+    public static IEnumerable<ILayer<T>> CreateDefaultBayesianNeuralNetworkLayers(NeuralNetworkArchitecture<T> architecture)
+    {
+        if (architecture == null)
+        {
+            throw new ArgumentNullException(nameof(architecture));
+        }
+
+        int inputSize = architecture.GetInputShape()[0];
+        int outputSize = architecture.OutputSize;
+
+        if (inputSize <= 0)
+        {
+            throw new InvalidOperationException("Input size must be greater than zero.");
+        }
+
+        if (outputSize <= 0)
+        {
+            throw new InvalidOperationException("Output size must be greater than zero.");
+        }
+
+        List<int> hiddenLayerSizes = new List<int>();
+        switch (architecture.Complexity)
+        {
+            case NetworkComplexity.Simple:
+                hiddenLayerSizes.Add((inputSize + outputSize) / 2);
+                break;
+            case NetworkComplexity.Medium:
+                hiddenLayerSizes.Add(inputSize * 2);
+                hiddenLayerSizes.Add(inputSize);
+                break;
+            case NetworkComplexity.Deep:
+                hiddenLayerSizes.Add(inputSize * 2);
+                hiddenLayerSizes.Add(inputSize * 2);
+                hiddenLayerSizes.Add(inputSize);
+                break;
+            default:
+                hiddenLayerSizes.Add(inputSize);
+                break;
+        }
+
+        int firstHiddenLayerSize = hiddenLayerSizes.Count > 0 ? hiddenLayerSizes[0] : outputSize;
+        yield return new AiDotNet.UncertaintyQuantification.Layers.BayesianDenseLayer<T>(inputSize, firstHiddenLayerSize, new ReLUActivation<T>() as IActivationFunction<T>);
+        yield return new ActivationLayer<T>([firstHiddenLayerSize], new ReLUActivation<T>() as IActivationFunction<T>);
+
+        for (int i = 0; i < hiddenLayerSizes.Count - 1; i++)
+        {
+            int currentLayerSize = hiddenLayerSizes[i];
+            int nextLayerSize = hiddenLayerSizes[i + 1];
+
+            yield return new AiDotNet.UncertaintyQuantification.Layers.BayesianDenseLayer<T>(currentLayerSize, nextLayerSize, new ReLUActivation<T>() as IActivationFunction<T>);
+            yield return new ActivationLayer<T>([nextLayerSize], new ReLUActivation<T>() as IActivationFunction<T>);
+        }
+
+        if (hiddenLayerSizes.Count > 0)
+        {
+            int lastHiddenLayerSize = hiddenLayerSizes[hiddenLayerSizes.Count - 1];
+            yield return new AiDotNet.UncertaintyQuantification.Layers.BayesianDenseLayer<T>(lastHiddenLayerSize, outputSize, new SoftmaxActivation<T>() as IActivationFunction<T>);
+        }
+        else
+        {
+            yield return new AiDotNet.UncertaintyQuantification.Layers.BayesianDenseLayer<T>(inputSize, outputSize, new SoftmaxActivation<T>() as IActivationFunction<T>);
+        }
+
+        yield return new ActivationLayer<T>(new[] { outputSize }, new SoftmaxActivation<T>() as IActivationFunction<T>);
+    }
+
     /// <summary>
     /// Creates a default configuration of layers for a Liquid State Machine (LSM) neural network.
     /// </summary>
@@ -2289,4 +2362,4 @@ public static IEnumerable<ILayer<T>> CreateDefaultFeedForwardLayers(
 
         yield return new DenseLayer<T>(hiddenLayerSize, architecture.OutputSize, outputActivation);
     }
-}
+}

src/Models/Options/UncertaintyQuantificationOptions.cs

@@ -75,4 +75,20 @@ public sealed class UncertaintyQuantificationOptions
     /// calibrated probabilities as the prediction output from uncertainty APIs.
     /// </remarks>
     public bool EnableTemperatureScaling { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets the number of independently trained models used for deep ensemble uncertainty estimation.
+    /// </summary>
+    /// <remarks>
+    /// This value is only used when <see cref="Method"/> is <see cref="UncertaintyQuantificationMethod.DeepEnsemble"/>.
+    /// </remarks>
+    public int DeepEnsembleSize { get; set; } = 5;
+
+    /// <summary>
+    /// Gets or sets the standard deviation of the initial parameter perturbation applied when constructing ensemble members.
+    /// </summary>
+    /// <remarks>
+    /// This helps ensure ensemble members do not collapse to identical solutions when created from a shared base model.
+    /// </remarks>
+    public double DeepEnsembleInitialNoiseStdDev { get; set; } = 0.01;
 }

src/Models/Results/PredictionModelResult.cs

@@ -447,6 +447,14 @@ internal void SetUncertaintyCalibrationArtifacts(UncertaintyCalibrationArtifacts
         TemperatureScalingTemperature = artifacts.TemperatureScalingTemperature;
     }
 
+    internal void SetDeepEnsembleModels(List<IFullModel<T, TInput, TOutput>> models)
+    {
+        _deepEnsembleModels = models ?? throw new ArgumentNullException(nameof(models));
+    }
+
+    [JsonProperty]
+    private List<IFullModel<T, TInput, TOutput>>? _deepEnsembleModels;
+
     /// <summary>
     /// Initializes a new instance of the PredictionModelResult class with default values.
     /// </summary>
@@ -562,7 +570,7 @@ public TOutput Predict(TInput newData)
     /// </summary>
     /// <param name="newData">Input data to predict.</param>
     /// <param name="numSamples">Optional number of stochastic samples to draw (overrides configured defaults).</param>
-    /// <returns>A tuple containing the mean prediction and an uncertainty estimate (variance).</returns>
+    /// <returns>An uncertainty-augmented prediction result.</returns>
     /// <remarks>
     /// <para>
     /// When uncertainty quantification is enabled, this method performs multiple stochastic forward passes and
@@ -610,6 +618,16 @@ public UncertaintyPredictionResult<T, TOutput> PredictWithUncertainty(TInput new
             return PredictWithConformal(newData, uq, method);
         }
 
+        if (method == UncertaintyQuantificationMethod.DeepEnsemble)
+        {
+            return PredictWithDeepEnsemble(newData, uq, method);
+        }
+
+        if (method == UncertaintyQuantificationMethod.BayesianNeuralNetwork)
+        {
+            return PredictWithBayesianNeuralNetwork(newData, uq, method);
+        }
+
         var effectiveSamples = numSamples ?? uq.NumSamples;
         if (effectiveSamples < 1)
         {
@@ -637,7 +655,9 @@ public UncertaintyPredictionResult<T, TOutput> PredictWithUncertainty(TInput new
             {
                 var (meanTensor, varianceTensor, predictiveEntropy, mutualInformation) = ComputeMonteCarloMomentsAndMetrics(
                     normalizedNewData,
-                    effectiveSamples);
+                    effectiveSamples,
+                    mcDropoutLayers,
+                    uq.RandomSeed);
 
                 var meanOutput = ConvertFromTensor(meanTensor);
                 var denormalizedMean = NormalizationInfo.Normalizer.Denormalize(meanOutput, NormalizationInfo.YParams);
@@ -713,6 +733,162 @@ private UncertaintyPredictionResult<T, TOutput> PredictWithConformal(
             classificationSet: classificationSet);
     }
 
+    private UncertaintyPredictionResult<T, TOutput> PredictWithDeepEnsemble(
+        TInput newData,
+        UncertaintyQuantificationOptions uq,
+        UncertaintyQuantificationMethod method)
+    {
+        if (_deepEnsembleModels is not { Count: > 0 })
+        {
+            var deterministic = Predict(newData);
+            var fallbackMetrics = CreateDefaultUncertaintyMetrics(deterministic, mutualInformation: null);
+            return new UncertaintyPredictionResult<T, TOutput>(
+                methodUsed: method,
+                prediction: deterministic,
+                variance: CreateZeroLike(deterministic),
+                metrics: fallbackMetrics);
+        }
+
+        var (normalizedNewData, _) = NormalizationInfo.Normalizer!.NormalizeInput(newData);
+
+        var numOps = MathHelper.GetNumericOperations<T>();
+        var samples = new List<Tensor<T>>(_deepEnsembleModels.Count);
+
+        for (int i = 0; i < _deepEnsembleModels.Count; i++)
+        {
+            var normalizedPrediction = _deepEnsembleModels[i].Predict(normalizedNewData);
+            samples.Add(ConversionsHelper.ConvertToTensor<T>(normalizedPrediction!).Clone());
+        }
+
+        var first = samples[0];
+        var firstVector = first.ToVector();
+        var (treatAsProbabilities, batch, classes) = InferProbabilityDistributionLayout(first, firstVector);
+
+        if (treatAsProbabilities && classes > 1 && HasTemperatureScaling)
+        {
+            for (int i = 0; i < samples.Count; i++)
+            {
+                samples[i] = ApplyTemperatureScalingToProbabilityTensor(samples[i], TemperatureScalingTemperature, batch, classes);
+            }
+        }
+
+        var (meanTensor, varianceTensor) = ComputeMeanAndVariance(samples);
+
+        var meanOutput = ConvertFromTensor(meanTensor);
+        var denormalizedMean = NormalizationInfo.Normalizer!.Denormalize(meanOutput, NormalizationInfo.YParams);
+
+        if (uq.DenormalizeUncertainty)
+        {
+            varianceTensor = DenormalizeVarianceIfSupported(varianceTensor, NormalizationInfo.YParams);
+        }
+
+        var varianceOutput = ConvertFromTensor(varianceTensor);
+
+        var predictiveEntropy = CreateZeroVectorTensor(batch);
+        var mutualInformation = CreateZeroVectorTensor(batch);
+
+        if (treatAsProbabilities && classes > 1)
+        {
+            var expectedEntropySum = new Vector<T>(batch);
+            foreach (var sample in samples)
+            {
+                var sampleEntropy = ComputePerSampleEntropy(sample.ToVector(), batch, classes);
+                for (int b = 0; b < batch; b++)
+                {
+                    expectedEntropySum[b] = numOps.Add(expectedEntropySum[b], sampleEntropy[b]);
+                }
+            }
+
+            var meanVector = meanTensor.ToVector();
+            var predictiveEntropyVec = ComputePerSampleEntropy(meanVector, batch, classes);
+            var expectedEntropyVec = new Vector<T>(batch);
+            for (int b = 0; b < batch; b++)
+            {
+                expectedEntropyVec[b] = numOps.Divide(expectedEntropySum[b], numOps.FromDouble(samples.Count));
+            }
+
+            var miVec = new Vector<T>(batch);
+            for (int b = 0; b < batch; b++)
+            {
+                var mi = numOps.Subtract(predictiveEntropyVec[b], expectedEntropyVec[b]);
+                if (numOps.LessThan(mi, numOps.Zero))
+                {
+                    mi = numOps.Zero;
+                }
+                miVec[b] = mi;
+            }
+
+            predictiveEntropy = new Tensor<T>([batch], predictiveEntropyVec);
+            mutualInformation = new Tensor<T>([batch], miVec);
+        }
+
+        var metrics = CreateDefaultUncertaintyMetrics(denormalizedMean, mutualInformation);
+        metrics[PredictiveEntropyMetricKey] = predictiveEntropy;
+        metrics[MutualInformationMetricKey] = mutualInformation;
+
+        return new UncertaintyPredictionResult<T, TOutput>(
+            methodUsed: method,
+            prediction: denormalizedMean,
+            variance: varianceOutput,
+            metrics: metrics);
+    }
+
+    private UncertaintyPredictionResult<T, TOutput> PredictWithBayesianNeuralNetwork(
+        TInput newData,
+        UncertaintyQuantificationOptions uq,
+        UncertaintyQuantificationMethod method)
+    {
+        var estimator = Model as AiDotNet.UncertaintyQuantification.Interfaces.IUncertaintyEstimator<T>;
+        if (estimator == null)
+        {
+            var deterministic = Predict(newData);
+            var fallbackMetrics = CreateDefaultUncertaintyMetrics(deterministic, mutualInformation: null);
+            return new UncertaintyPredictionResult<T, TOutput>(
+                methodUsed: method,
+                prediction: deterministic,
+                variance: CreateZeroLike(deterministic),
+                metrics: fallbackMetrics);
+        }
+
+        var (normalizedNewData, _) = NormalizationInfo.Normalizer!.NormalizeInput(newData);
+        if (normalizedNewData == null)
+        {
+            throw new InvalidOperationException("Normalizer returned null input.");
+        }
+        var inputTensor = ConversionsHelper.ConvertToTensor<T>(normalizedNewData).Clone();
+
+        var uqResult = estimator.PredictWithUncertainty(inputTensor);
+
+        var meanOutput = ConvertFromTensor(uqResult.Prediction);
+        var denormalizedMean = NormalizationInfo.Normalizer!.Denormalize(meanOutput, NormalizationInfo.YParams);
+
+        var varianceTensor = uqResult.Variance != null
+            ? uqResult.Variance.Clone()
+            : new Tensor<T>(
+                uqResult.Prediction.Shape,
+                Vector<T>.CreateDefault(uqResult.Prediction.Length, MathHelper.GetNumericOperations<T>().Zero));
+        if (uq.DenormalizeUncertainty)
+        {
+            varianceTensor = DenormalizeVarianceIfSupported(varianceTensor, NormalizationInfo.YParams);
+        }
+        var varianceOutput = ConvertFromTensor(varianceTensor);
+
+        var metrics = CreateDefaultUncertaintyMetrics(denormalizedMean, mutualInformation: null);
+        if (uqResult.Metrics != null)
+        {
+            foreach (var kvp in uqResult.Metrics)
+            {
+                metrics[kvp.Key] = kvp.Value;
+            }
+        }
+
+        return new UncertaintyPredictionResult<T, TOutput>(
+            methodUsed: method,
+            prediction: denormalizedMean,
+            variance: varianceOutput,
+            metrics: metrics);
+    }
+
     private static (int batch, int classes) InferBatchAndClasses(Tensor<T> probabilities, int configuredClasses)
     {
         var classes = configuredClasses > 0
@@ -839,13 +1015,22 @@ private static Tensor<T> CreateZeroVectorTensor(int length)
     }
 
     private (Tensor<T> mean, Tensor<T> variance, Tensor<T> predictiveEntropy, Tensor<T> mutualInformation)
-        ComputeMonteCarloMomentsAndMetrics(TInput normalizedNewData, int numSamples)
+        ComputeMonteCarloMomentsAndMetrics(
+            TInput normalizedNewData,
+            int numSamples,
+            IReadOnlyList<AiDotNet.UncertaintyQuantification.Layers.MCDropoutLayer<T>> mcDropoutLayers,
+            int? randomSeed)
     {
         var numOps = MathHelper.GetNumericOperations<T>();
 
         var samples = new List<Tensor<T>>(capacity: numSamples);
         for (int i = 0; i < numSamples; i++)
         {
+            if (randomSeed.HasValue)
+            {
+                ResetMonteCarloDropoutRng(mcDropoutLayers, randomSeed.Value, i);
+            }
+
             var normalizedPrediction = Model!.Predict(normalizedNewData);
             samples.Add(ConversionsHelper.ConvertToTensor<T>(normalizedPrediction!).Clone());
         }
@@ -905,6 +1090,24 @@ private static Tensor<T> CreateZeroVectorTensor(int length)
         return (meanTensor, varianceTensor, predictiveEntropy, mutualInformation);
     }
 
+    private static void ResetMonteCarloDropoutRng(
+        IReadOnlyList<AiDotNet.UncertaintyQuantification.Layers.MCDropoutLayer<T>> layers,
+        int baseSeed,
+        int sampleIndex)
+    {
+        unchecked
+        {
+            for (int i = 0; i < layers.Count; i++)
+            {
+                // Mix base seed, sample index, and layer index to get deterministic but distinct streams.
+                var mixed = baseSeed;
+                mixed = (mixed * 1000003) ^ (sampleIndex + 1);
+                mixed = (mixed * 1009) ^ (i + 1);
+                layers[i].ResetRng(mixed);
+            }
+        }
+    }
+
     private TOutput ApplyTemperatureScalingToOutputProbabilities(TOutput outputProbabilities, T temperature)
     {
         var tensor = ConversionsHelper.ConvertToTensor<T>(outputProbabilities!).Clone();

src/NeuralNetworks/Layers/ActivationLayer.cs

@@ -5,7 +5,7 @@ namespace AiDotNet.NeuralNetworks.Layers;
 /// <para>
 /// Activation functions introduce non-linearity to neural networks. Non-linearity means the output isn't 
 /// simply proportional to the input (like y = 2x). Instead, it can follow curves or more complex patterns.
-/// Without non-linearity, a neural network—no matter how many layers—would behave just like a single layer,
+/// Without non-linearity, a neural network�no matter how many layers�would behave just like a single layer,
 /// severely limiting what it can learn.
 /// </para>
 /// <para>
@@ -304,8 +304,9 @@ private Tensor<T> ApplyVectorActivation(Tensor<T> input)
     /// </remarks>
     private Tensor<T> BackwardScalarActivation(Tensor<T> outputGradient)
     {
-        return _lastInput!.Transform((x, indices) => 
-            NumOps.Multiply(ScalarActivation!.Derivative(x), outputGradient[indices]));
+        var gradVector = outputGradient.ToVector();
+        return _lastInput!.Transform((x, i) =>
+            NumOps.Multiply(ScalarActivation!.Derivative(x), gradVector[i]));
     }
 
 
@@ -467,4 +468,4 @@ public override void ResetState()
     {
         _lastInput = null;
     }
-}
+}