Fix UCME (#118)

Author: devmotion

Project.toml

@@ -1,7 +1,7 @@
 name = "CalibrationErrors"
 uuid = "33913031-fe46-5864-950f-100836f47845"
 authors = ["David Widmann <david.widmann@it.uu.se>"]
-version = "0.5.21"
+version = "0.5.22"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"

src/ucme.jl

@@ -92,9 +92,8 @@ function unsafe_ucme_eval(
     return res
 end
 function unsafe_ucme_eval(kernel::Kernel, p::Real, y::Bool, testp::Real, testy::Bool)
-    noty = !y
-    return (y - p) * kernel((p, y), (testp, testy)) +
-           (noty - p) * kernel((p, noty), (testp, testy))
+    return (kernel((p, true), (testp, testy)) - kernel((p, false), (testp, testy))) *
+           (y - p)
 end
 
 function unsafe_ucme_eval(kernel::KernelTensorProduct, p, y, testp, testy)
@@ -126,15 +125,12 @@ function unsafe_ucme_eval_targets(
     testp::AbstractVector{<:Real},
     testy::Integer,
 )
-    res = sum(((z == y) - pz) * kernel(z, testy) for (z, pz) in enumerate(p))
-    return res
+    return sum(((z == y) - pz) * kernel(z, testy) for (z, pz) in enumerate(p))
 end
 function unsafe_ucme_eval_targets(
     kernel::Kernel, p::Real, y::Bool, testp::Real, testy::Bool
 )
-    noty = !y
-    res = (y - p) * kernel(y, testy) + (noty - p) * kernel(noty, testy)
-    return res
+    return (kernel(true, testy) - kernel(false, testy)) * (y - p)
 end
 
 function unsafe_ucme_eval_targets(
@@ -144,11 +140,10 @@ function unsafe_ucme_eval_targets(
     testp::AbstractVector{<:Real},
     testy::Integer,
 )
-    @inbounds res = (y == testy) - p[testy]
-    return res
+    return @inbounds (y == testy) - p[testy]
 end
 function unsafe_ucme_eval_targets(
     kernel::WhiteKernel, p::Real, y::Bool, testp::Real, testy::Bool
 )
-    return (2 * testy - 1) * (y - p)
+    return (testy - !testy) * (y - p)
 end

test/binning/medianvariance.jl

@@ -27,7 +27,7 @@
             @test sum(bin -> bin.nsamples, bins) == nsamples
             @test sum(bin -> bin.nsamples .* bin.mean_predictions, bins) ≈ sum(predictions)
             @test sum(bin -> bin.nsamples .* bin.proportions_targets, bins) ≈
-                  counts(targets, nclasses)
+                counts(targets, nclasses)
         end
 
         # set maximum number of bins
@@ -47,7 +47,7 @@
             @test sum(bin -> bin.nsamples, bins) == nsamples
             @test sum(bin -> bin.nsamples .* bin.mean_predictions, bins) ≈ sum(predictions)
             @test sum(bin -> bin.nsamples .* bin.proportions_targets, bins) ≈
-                  counts(targets, nclasses)
+                counts(targets, nclasses)
         end
     end
 
@@ -102,7 +102,7 @@
             @test bins[i].nsamples == length(idxs)
             @test bins[i].mean_predictions ≈ mean(predictions[idxs])
             @test bins[i].proportions_targets ==
-                  vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
+                vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
         end
 
         bins = CalibrationErrors.perform(MedianVarianceBinning(3), predictions, targets)

test/binning/uniform.jl

@@ -98,7 +98,7 @@
             @test bins[i].nsamples == length(idxs)
             @test bins[i].mean_predictions == mean(predictions[idxs])
             @test bins[i].proportions_targets ==
-                  vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
+                vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
         end
 
         bins = CalibrationErrors.perform(UniformBinning(1), predictions, targets)
@@ -134,7 +134,7 @@
             @test bins[i].nsamples == length(idxs)
             @test bins[i].mean_predictions == mean(predictions[idxs])
             @test bins[i].proportions_targets ==
-                  vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
+                vec(mean(Matrix{Float64}(I, 3, 3)[:, targets[idxs]]; dims=2))
         end
 
         bins = CalibrationErrors.perform(UniformBinning(1), predictions, targets)

test/deprecated.jl

@@ -20,9 +20,9 @@
         for estimator in (ece, skce1, skce2, skce3, ucme)
             estimate = estimator(predictions, targets)
             @test @test_deprecated(calibrationerror(estimator, predictions, targets)) ==
-                  estimate
+                estimate
             @test @test_deprecated(calibrationerror(estimator, (predictions, targets))) ==
-                  estimate
+                estimate
             @test @test_deprecated(
                 calibrationerror(estimator, reduce(hcat, predictions), targets)
             ) == estimate

test/skce/generic.jl

@@ -32,7 +32,7 @@
 
         @test unsafe_skce_eval(kernelfull ⊗ WhiteKernel(), pfull, yint, p̃full, ỹint) ≈ val
         @test unsafe_skce_eval(kernelfull ⊗ WhiteKernel2(), pfull, yint, p̃full, ỹint) ≈
-              val
+            val
         @test unsafe_skce_eval(
             TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, p̃full, ỹint
         ) ≈ val

test/ucme.jl

@@ -93,37 +93,44 @@
     end
 
     @testset "binary classification" begin
-        # probabilities and boolean targets
+        # probabilities and corresponding full categorical distribution
         p, testp = rand(2)
-        y, testy = rand(Bool, 2)
-        scale = rand()
-        kernel = SqExponentialKernel() ∘ ScaleTransform(scale)
-        val = unsafe_ucme_eval(kernel ⊗ WhiteKernel(), p, y, testp, testy)
-        @test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
-        @test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy) ≈
-              val
-        @test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy) ≈
-              val
-
-        # corresponding values and kernel for full categorical distribution
         pfull = [p, 1 - p]
-        yint = 2 - y
         testpfull = [testp, 1 - testp]
-        testyint = 2 - testy
+
+        # kernel for probabilities and corresponding one for full categorical distributions
+        scale = rand()
+        kernel = SqExponentialKernel() ∘ ScaleTransform(scale)
         kernelfull = SqExponentialKernel() ∘ ScaleTransform(scale / sqrt(2))
 
-        @test unsafe_ucme_eval(
-            kernelfull ⊗ WhiteKernel(), pfull, yint, testpfull, testyint
-        ) ≈ val
-        @test unsafe_ucme_eval(
-            kernelfull ⊗ WhiteKernel2(), pfull, yint, testpfull, testyint
-        ) ≈ val
-        @test unsafe_ucme_eval(
-            TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, testpfull, testyint
-        ) ≈ val
-        @test unsafe_ucme_eval(
-            TensorProduct2(kernelfull, WhiteKernel2()), pfull, yint, testpfull, testyint
-        ) ≈ val
+        # for different targets
+        for y in (true, false), testy in (true, false)
+            # check values for probabilities
+            val = unsafe_ucme_eval(kernel ⊗ WhiteKernel(), p, y, testp, testy)
+            @test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
+            @test unsafe_ucme_eval(
+                TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy
+            ) ≈ val
+            @test unsafe_ucme_eval(
+                TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy
+            ) ≈ val
+
+            # check values for categorical distributions
+            yint = 2 - y
+            testyint = 2 - testy
+            @test unsafe_ucme_eval(
+                kernelfull ⊗ WhiteKernel(), pfull, yint, testpfull, testyint
+            ) ≈ val
+            @test unsafe_ucme_eval(
+                kernelfull ⊗ WhiteKernel2(), pfull, yint, testpfull, testyint
+            ) ≈ val
+            @test unsafe_ucme_eval(
+                TensorProduct2(kernelfull, WhiteKernel()), pfull, yint, testpfull, testyint
+            ) ≈ val
+            @test unsafe_ucme_eval(
+                TensorProduct2(kernelfull, WhiteKernel2()), pfull, yint, testpfull, testyint
+            ) ≈ val
+        end
     end
 
     @testset "multi-class classification" begin
@@ -140,8 +147,8 @@
 
         @test unsafe_ucme_eval(kernel ⊗ WhiteKernel2(), p, y, testp, testy) ≈ val
         @test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel()), p, y, testp, testy) ≈
-              val
+            val
         @test unsafe_ucme_eval(TensorProduct2(kernel, WhiteKernel2()), p, y, testp, testy) ≈
-              val
+            val
     end
 end