WeightDecay before ClipNorm

Author: prbzrg

examples/usage.jl

@@ -84,8 +84,8 @@ if !isfile(icnf_mach_fn)
         icnf,
         optimizers = (
             OptimiserChain(
-                ClipNorm(1.0f0, 2.0f0; throw = true),
                 WeightDecay(; lambda = 1.0f-2),
+                ClipNorm(1.0f0, 2.0f0; throw = true),
                 Adam(; eta = 1.0f-3, beta = (9.0f-1, 9.99f-1), epsilon = eps(Float32)),
             ),
         ),

src/exts/mlj_ext/core_cond_icnf.jl

@@ -12,12 +12,12 @@ function CondICNFModel(;
     loss::Function = loss,
     optimizers::Tuple = (
         Optimisers.OptimiserChain(
+            Optimisers.WeightDecay(; lambda = convert(eltype(icnf), 1.0e-2)),
             Optimisers.ClipNorm(
                 one(eltype(icnf)),
                 convert(eltype(icnf), 2.0e0);
                 throw = true,
             ),
-            Optimisers.WeightDecay(; lambda = convert(eltype(icnf), 1.0e-2)),
             Optimisers.Adam(;
                 eta = convert(eltype(icnf), 1.0e-3),
                 beta = (convert(eltype(icnf), 9.0e-1), convert(eltype(icnf), 9.99e-1)),

src/exts/mlj_ext/core_icnf.jl

@@ -12,12 +12,12 @@ function ICNFModel(;
     loss::Function = loss,
     optimizers::Tuple = (
         Optimisers.OptimiserChain(
+            Optimisers.WeightDecay(; lambda = convert(eltype(icnf), 1.0e-2)),
             Optimisers.ClipNorm(
                 one(eltype(icnf)),
                 convert(eltype(icnf), 2.0e0);
                 throw = true,
             ),
-            Optimisers.WeightDecay(; lambda = convert(eltype(icnf), 1.0e-2)),
             Optimisers.Adam(;
                 eta = convert(eltype(icnf), 1.0e-3),
                 beta = (convert(eltype(icnf), 9.0e-1), convert(eltype(icnf), 9.99e-1)),