Merge pull request #38 from LAMPSPUC/dev

Version 0.1.5

Author: Giovanni3A

Project.toml

@@ -1,7 +1,7 @@
 name = "ApplicationDrivenLearning"
 uuid = "0856f1c8-ef17-4e14-9230-2773e47a789e"
 authors = ["Giovanni Amorim", "Joaquim Garcia"]
-version = "0.1.4"
+version = "0.1.5"
 
 [deps]
 BilevelJuMP = "485130c0-026e-11ea-0f1a-6992cd14145c"

src/ApplicationDrivenLearning.jl

@@ -198,8 +198,6 @@ function train!(
     elseif options.mode == GradientMPIMode
         return train_with_gradient_mpi!(model, X, y, options.params)
     elseif options.mode == BilevelMode
-        asr_msg = "BilevelMode not implemented for multiple forecasting models"
-        @assert length(model.forecast.networks) == 1 asr_msg
         return solve_bilevel(model, X, y, options.params)
     else
         # should never get here

src/jump.jl

@@ -136,7 +136,9 @@ function JuMP.set_optimizer(model::Model, builder, evaluate_duals::Bool = true)
     JuMP.set_optimizer(model.plan, () -> new_diff_optimizer)
 
     # basic setting for assess model
-    return JuMP.set_optimizer(model.assess, builder)
+    JuMP.set_optimizer(model.assess, builder)
+
+    return nothing
 end
 
 function JuMP.set_silent(model::Model)

src/optimizers/bilevel.jl

@@ -117,46 +117,49 @@ function solve_bilevel(
     end
 
     # implement predictive model expression iterating through 
-    # layers and creating predictive expression
-    layers_inpt = Dict{Any,Any}(
-        output_idx => X[1:T, input_idx] for
-        (input_idx, output_idx) in model.forecast.input_output_map[1]
-    )
-    predictive_model_vars = Dict{Int,Any}()
-    i_layer = 1
-    for layer in model.forecast.networks[1]
-        # if it is layer with parameters, process output
-        if has_params(layer)
-            # get size and parameters W and b
-            (layer_size_out, layer_size_in) = size(layer.weight)
-            W = @variable(
-                Upper(bilevel_model),
-                [1:layer_size_out, 1:layer_size_in]
-            )
-            if layer.bias == false
-                b = zeros(layer_size_out)
+    # models and layers to create predictive expression
+    npreds = size(model.forecast.networks, 1)
+    predictive_model_vars = [Dict{Int,Any}() for ipred = 1:npreds]
+    y_hat = Matrix{Any}(undef, size(Y, 1), size(Y, 2))
+    for ipred = 1:npreds
+        layers_inpt = Dict{Any,Any}(
+            output_idx => X[1:T, input_idx] for (input_idx, output_idx) in
+            model.forecast.input_output_map[ipred]
+        )
+        i_layer = 1
+        for layer in model.forecast.networks[ipred]
+            # if it is layer with parameters, process output
+            if has_params(layer)
+                # get size and parameters W and b
+                (layer_size_out, layer_size_in) = size(layer.weight)
+                W = @variable(
+                    Upper(bilevel_model),
+                    [1:layer_size_out, 1:layer_size_in]
+                )
+                if layer.bias == false
+                    b = zeros(layer_size_out)
+                else
+                    b = @variable(Upper(bilevel_model), [1:layer_size_out])
+                end
+                predictive_model_vars[ipred][i_layer] = Dict(:W => W, :b => b)
+                # build layer output as next layer input
+                for output_idx in values(model.forecast.input_output_map[ipred])
+                    layers_inpt[output_idx] =
+                        layer.σ(W * layers_inpt[output_idx]' .+ b)'
+                end
+                # if activation function layer, just apply
+            elseif supertype(typeof(layer)) == Function
+                for output_idx in values(model.forecast.input_output_map[ipred])
+                    layers_inpt[output_idx] = layer(layers_inpt[output_idx])
+                end
             else
-                b = @variable(Upper(bilevel_model), [1:layer_size_out])
-            end
-            predictive_model_vars[i_layer] = Dict(:W => W, :b => b)
-            # build layer output as next layer input
-            for output_idx in values(model.forecast.input_output_map[1])
-                layers_inpt[output_idx] =
-                    layer.σ(W * layers_inpt[output_idx]' .+ b)'
+                println("Network $ipred layer $ilayer type not supported")
             end
-            # if activation function layer, just apply
-        elseif supertype(typeof(layer)) == Function
-            for output_idx in values(model.forecast.input_output_map[1])
-                layers_inpt[output_idx] = layer(layers_inpt[output_idx])
-            end
-        else
-            println("Network layer $ilayer type not supported")
+            i_layer += 1
+        end
+        for (output_idx, prediction) in layers_inpt
+            y_hat[:, output_idx] = prediction
         end
-        i_layer += 1
-    end
-    y_hat = Matrix{Any}(undef, size(Y, 1), size(Y, 2))
-    for (output_idx, prediction) in layers_inpt
-        y_hat[:, output_idx] = prediction
     end
 
     # and apply prediction on lower model as constraint
@@ -174,17 +177,19 @@ function solve_bilevel(
     optimize!(bilevel_model)
 
     # fix parameters to predictive_model
-    ilayer = 1
-    for layer in model.forecast.networks[1]
-        if has_params(layer)
-            for p in Flux.trainables(layer.weight)
-                p .= value.(predictive_model_vars[ilayer][:W])
-            end
-            for p in Flux.trainables(layer.bias)
-                p .= value.(predictive_model_vars[ilayer][:b])
+    for ipred = 1:npreds
+        ilayer = 1
+        for layer in model.forecast.networks[ipred]
+            if has_params(layer)
+                for p in Flux.trainables(layer.weight)
+                    p .= value.(predictive_model_vars[ipred][ilayer][:W])
+                end
+                for p in Flux.trainables(layer.bias)
+                    p .= value.(predictive_model_vars[ipred][ilayer][:b])
+                end
             end
+            ilayer += 1
         end
-        ilayer += 1
     end
 
     return Solution(