More changes, the new approach now works in some cases, more to investigate. More tests to implement, and other tests to fix, etc.

Author: Samuel-amap

src/PlantSimEngine.jl

@@ -108,7 +108,7 @@ export PreviousTimeStep
 export AbstractModel
 export ModelList, MultiScaleModel, TimestepMappedVariable
 export MultiScaleMapping
-export Orchestrator2, TimestepRange, Var_to, Var_from, ModelTimestepMapping
+export Orchestrator, TimestepRange, ModelTimestepMapping
 export RMSE, NRMSE, EF, dr
 export Status, TimeStepTable, status
 export init_status!

src/dependencies/dependency_graph.jl

@@ -6,16 +6,16 @@ mutable struct HardDependencyNode{T} <: AbstractDependencyNode
     dependency::NamedTuple
     missing_dependency::Vector{Int}
     scale::String
-    inputs
-    outputs
+    #inputs
+    #outputs
     parent::Union{Nothing,<:AbstractDependencyNode}
     children::Vector{HardDependencyNode}
 end
 
 mutable struct TimestepMapping
     variable_from::Symbol
     variable_to::Symbol
-    node_to # SoftDependencyNode causes a circular reference, removing it as a shortcut TODO
+    timestep_to::Period
     mapping_function::Function
     mapping_data_template
     mapping_data::Dict{Int, Any} # TODO fix type stability : Int is the node id, Any is a vector of n elements of the variable's type, n being the # of required timesteps
@@ -26,8 +26,8 @@ mutable struct SoftDependencyNode{T} <: AbstractDependencyNode
     value::T
     process::Symbol
     scale::String
-    inputs
-    outputs
+    #inputs
+    #outputs
     hard_dependency::Vector{HardDependencyNode}
     parent::Union{Nothing,Vector{SoftDependencyNode}}
     parent_vars::Union{Nothing,NamedTuple}

src/dependencies/hard_dependencies.jl

@@ -100,8 +100,8 @@ function initialise_all_as_hard_dependency_node(models, scale)
             NamedTuple(),
             Int[],
             scale,
-            inputs_(i),
-            outputs_(i),
+            #inputs_(i),
+            #outputs_(i),
             nothing,
             HardDependencyNode[]
         ) for (p, i) in pairs(models)
@@ -134,7 +134,7 @@ Return a NamedTuple with the variables and their default values.
 The `vars_mapping` is a dictionary with the organ type as key and a dictionary as value. It is 
 computed from the user mapping like so:
 """
-function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator2=Orchestrator2())
+function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orchestrator::Orchestrator=Orchestrator())
     node_vars = variables(node) # e.g. (inputs = (:var1=-Inf, :var2=-Inf), outputs = (:var3=-Inf,))
     ins = node_vars.inputs
     ins_variables = keys(ins)
@@ -154,36 +154,22 @@ function variables_multiscale(node, organ, vars_mapping, st=NamedTuple(), orches
                 # If the variable is an output, we use the default value given by the model:
                 default = defaults[var]
             end
-
-            # TODO no idea how this meshes with refvector situations or previoustimestep
-#            if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true)                
-#                push!(vars_, var => default)
-#                if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
-#                        organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
-#                        push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
-#                    end
-#            else
-            if is_timestep_mapped(organ => var, orchestrator, search_inputs_only=true)
-                
-                push!(vars_, var => default)
-            else
-                if haskey(vars_mapping[organ], var)
-                    organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var])
-                    push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default))
-                    #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current 
-                    #* values, and another one the previous values.
-                    if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
-                        organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
-                        push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
-                    end
-                elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
-                    # If not found in the current time step, we check if the variable is mapped to the previous time step:
+            if haskey(vars_mapping[organ], var)
+                organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][var])
+                push!(vars_, var => MappedVar(organ_mapped, var, organ_mapped_var, default))
+                #* We still check if the variable also exists wrapped in PreviousTimeStep, because one model could use the current 
+                #* values, and another one the previous values.
+                if haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
                     organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
                     push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
-                else
-                    # Else we take the default value:
-                    push!(vars_, var => default)
                 end
+            elseif haskey(vars_mapping[organ], PreviousTimeStep(var, node.process))
+                # If not found in the current time step, we check if the variable is mapped to the previous time step:
+                organ_mapped, organ_mapped_var = _node_mapping(vars_mapping[organ][PreviousTimeStep(var, node.process)])
+                push!(vars_, var => MappedVar(organ_mapped, PreviousTimeStep(var, node.process), organ_mapped_var, default))
+            else
+                # Else we take the default value:
+                push!(vars_, var => default)
             end
         end
 #        end
@@ -210,12 +196,12 @@ function extract_timestep_mapped_outputs(m::MultiScaleModel, organ::String, outp
         key = process(m.model)
         extra_outputs = timestep_mapped_outputs_(m)
         #ind = findfirst(x -> first(x) == key, outputs_process[organ][key])
-        timestep_mapped_outputs_process[organ][key] = (; extra_outputs...) #TODO 
+        timestep_mapped_outputs_process[organ][key] = extra_outputs #TODO 
     end
 end
 
 # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children):
-function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator2=Orchestrator2()) where {T}
+function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestrator::Orchestrator=Orchestrator()) where {T}
     full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping)
     soft_dep_graphs = Dict{String,Any}()
     not_found = Dict{Symbol,DataType}()
@@ -239,7 +225,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr
     #* dependency may not appear in its own scale, but this is treated in the soft-dependency computation
     inputs_process = Dict{String,Dict{Symbol,Vector}}()
     outputs_process = Dict{String,Dict{Symbol,Vector}}()
-    timestep_mapped_outputs_process = Dict{String,Dict{Symbol,NamedTuple}}()
+    timestep_mapped_outputs_process = Dict{String,Dict{Symbol,Vector}}()
     for (organ, model) in mapping
         # Get the status given by the user, that is used to set the default values of the variables in the mapping:
         st_scale_user = get_status(model)
@@ -324,8 +310,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr
                     dep_node_model.dependency,
                     dep_node_model.missing_dependency,
                     dep_node_model.scale,
-                    dep_node_model.inputs,
-                    dep_node_model.outputs,
+                    #dep_node_model.inputs,
+                    #dep_node_model.outputs,
                     parent_node,
                     dep_node_model.children
                 )
@@ -398,8 +384,8 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr
                 soft_dep_vars.value,
                 process_, # process name
                 organ, # scale
-                inputs_process[organ][process_], # These are the inputs, potentially multiscale
-                outputs_process[organ][process_], # Same for outputs
+                #inputs_process[organ][process_], # These are the inputs, potentially multiscale
+                #outputs_process[organ][process_], # Same for outputs
                 AbstractTrees.children(soft_dep_vars), # hard dependencies
                 nothing,
                 nothing,
@@ -430,7 +416,7 @@ function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true, orchestr
             end
         end
 
-        soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ], timestep_mapped_outputs=haskey(timestep_mapped_outputs_process,organ) ? timestep_mapped_outputs_process[organ] : Dict{Symbol,NamedTuple}())
+        soft_dep_graphs[organ] = (soft_dep_graph=soft_dep_graph, inputs=inputs_process[organ], outputs=outputs_process[organ], timestep_mapped_outputs=haskey(timestep_mapped_outputs_process,organ) ? timestep_mapped_outputs_process[organ] : Dict{Symbol,Vector}())
         not_found = merge(not_found, hard_deps[organ].not_found)
     end
 

src/dependencies/soft_dependencies.jl

@@ -64,8 +64,8 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}},
             soft_dep_vars.value,
             process_, # process name
             "",
-            inputs_(soft_dep_vars.value),
-            outputs_(soft_dep_vars.value),
+            #inputs_(soft_dep_vars.value),
+            #outputs_(soft_dep_vars.value),
             AbstractTrees.children(soft_dep_vars), # hard dependencies
             nothing,
             nothing,
@@ -323,156 +323,32 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic
             end
 
             if haskey(timestep_mapped_outs, proc)
-                CreateTimeStepMapping(i, reverse_multiscale_mapping, organ, proc)
+                create_timestep_mapping(i, timestep_mapped_outs, proc)
             end
 
         end
     end
 
     dep_graph = DependencyGraph(independant_process_root, soft_dep_graphs_roots.not_found)
 
-    # TODO move these into the loop to directly act on the ndoe, use the provided timestep mappings
-    # Don't provide parents, and then change the run! function to apply from the node to itself
-    #traverse_dependency_graph!(x ->  set_non_default_timestep_in_node(x, orchestrator), dependency_graph, visit_hard_dep=false)
-    #traverse_dependency_graph!(x -> add_timestep_data_to_node(x, orchestrator), dep_graph, visit_hard_dep=false)
-
-
     return dep_graph
 end
 
-# # set the timestep for everyone first, else we might not use the correct timestep when looking at the parents later
-function set_non_default_timestep_in_node(soft_dependency_node, orchestrator::Orchestrator2)
-    for mtsm in orchestrator.non_default_timestep_mapping
-        if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value))
-            soft_dependency_node.timestep = mtsm.timestep
-        end
-    end
-end
-
-function add_timestep_data_to_node(soft_dependency_node, orchestrator::Orchestrator2)
-
-    # now we can create the mapping
-    for mtsm in orchestrator.non_default_timestep_mapping
-        if mtsm.scale == soft_dependency_node.scale && (mtsm.model) == typeof(model_(soft_dependency_node.value))
-            for (var_from, var_to) in mtsm.var_to_var        
-                if !isnothing(soft_dependency_node.parent)
-                    parent = nothing
-                    variable_mapping = nothing
-                    for parent_node in soft_dependency_node.parent
-                        if typeof(parent_node.value) == var_from.model && parent_node.scale == var_from.scale
-                            parent = parent_node                                            
-                            variable_mapping = create_timestep_mapping(soft_dependency_node, parent, var_to, var_from)
-                            break
-                        end
-                    end
-                    if isnothing(parent)
-                        #error
-                    end
-                    if isnothing(parent.timestep_mapping_data)
-                        parent.timestep_mapping_data = Vector{TimestepMapping}()
-                    end
-                    push!(parent.timestep_mapping_data, variable_mapping)
-                else
-                    # Error
-                end
-            end                
-        end
-    end
-end
-
-# Create the structure to attach to the softdependencynode, (not to be confused with the user-declared timestep-mapping structure)
-function create_timestep_mapping(node::SoftDependencyNode, parent::SoftDependencyNode, var_to::Var_to, var_from::Var_from)
-    
-    @assert parent.timestep != 0 "Error : node timestep internally set to 0"
-
-    timestep_ratio = node.timestep / parent.timestep
-
-    # Keeping things simple for now, only integers allowed
-    @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio"
-
-    # TODO ensure type compatibility between var_to and var_from
-    # Simplification probably possible by doing the check earlier
-
-    # TODO test previoustimestep
-    var_type = DataType
-    
-    for (symbol, var_dump) in node.inputs
-        for var in var_dump
-            if isa(var, MappedVar)
-                # check the source variable, because the sink one might be a vector...?
-                # TODO multinode mapping
-                if isa(var.source_organ, MultiNodeMapping)
-                    if var.source_variable[1] == var_from.name
-                        # This should be a fixed size array, ideally
-                        var_type = eltype(eltype(mapped_default(var)))
-                        break
-                    end
-                else
-                    if var.source_variable == var_from.name
-                        # This should be a fixed size array, ideally
-                        var_type = eltype(mapped_default(var))
-                        break
-                    end
-                end
-            else
-                if var.symbol == var_from.name
-                    @assert "untested"
-                    var_type = eltype(mapped_default(var))
-                    break
-                end
-            end
-        end
-    end
-
-    # TODO : type shouldn't be Any but Vector{var_type}
-    return TimestepMapping(var_from.name, var_to.name, node, var_from.mapping_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG, Any}())
-end
-
-
-function CreateTimeStepMapping(soft_dependency_node, reverse_multiscale_mapping, scale, proc)
-
-    for mapping_entry in reverse_multiscale_mapping[scale]
-        if isa(mapping_entry, MultiScaleModel)
-            if process(mapping_entry.model) == proc
-                for timestep_mapped_var_data in mapping_entry.timestep_mapped_variables
-                    timestep_ratio = timestep_mapped_var_data.timestep_to / soft_dependency_node.timestep
-                    if timestep_ratio > 1 # todo assert it's an int or a rational ?
-                        @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio"
+function create_timestep_mapping(soft_dependency_node, timestep_mapped_outs, proc)
 
-                        tmvd = timestep_mapped_var_data
+    for (proc_mapped, mapping_entries) in timestep_mapped_outs
 
-                        var_type = DataType
+        if proc_mapped == proc
+            for (var_to, (timestep_mapped_var_data, default_value)) in mapping_entries
+                timestep_ratio = timestep_mapped_var_data.timestep_to / soft_dependency_node.timestep
+                if timestep_ratio > 1 # todo assert it's an int or a rational ?
+                    @assert timestep_ratio == trunc(timestep_ratio) "Error : non-integer timestep ratio"
 
-                        for (symbol, var_dump) in soft_dependency_node.outputs
-                            for var in var_dump
-                                if isa(var, MappedVar)
-                                    # check the source variable, because the sink one might be a vector...?
-                                    # TODO multinode mapping
-                                    if isa(var.source_organ, MultiNodeMapping)
-                                        if var.source_variable[1] == tmvd.name_from
-                                            # This should be a fixed size array, ideally
-                                            var_type = eltype(eltype(mapped_default(var)))
-                                            break
-                                        end
-                                    else
-                                        if var.source_variable == tmvd.name_from
-                                            # This should be a fixed size array, ideally
-                                            var_type = eltype(mapped_default(var))
-                                            break
-                                        end
-                                    end
-                                else
-                                    if var.symbol == tmvd.name_from
-                                        @assert "untested"
-                                        var_type = eltype(mapped_default(var))
-                                        break
-                                    end
-                                end
-                            end
-                        end
+                    tmvd = timestep_mapped_var_data
+                    var_type = typeof(default_value)
 
-                        mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio))
-                        tsm = TimestepMapping(tmvd.name_from, tmvd.name_to, soft_dependency_node, tmvd.aggregation_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG,Any}())
+                    mapping_data_template = Vector{var_type}(undef, convert(Int64, timestep_ratio))
+                    tsm = TimestepMapping(tmvd.name_from, tmvd.name_to, timestep_mapped_var_data.timestep_to, tmvd.aggregation_function, mapping_data_template, Dict{MultiScaleTreeGraph.NodeMTG,Any}())
 
                     if isnothing(soft_dependency_node.timestep_mapping_data)
                         soft_dependency_node.timestep_mapping_data = Vector{TimestepMapping}()
@@ -483,10 +359,9 @@ function CreateTimeStepMapping(soft_dependency_node, reverse_multiscale_mapping,
         end
     end
 end
-end
 
 
-"""
+        """
     drop_process(proc_vars, process)
 
 Return a new `NamedTuple` with the process `process` removed from the `NamedTuple` `proc_vars`.
@@ -674,6 +549,13 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra
     return inputs_as_output_of_other_scale
 end
 
+function extract_mapped_outputs(timestep_mapped_outputs)
+    extracted = Pair[]
+    for pair in timestep_mapped_outputs
+        push!(extracted, Pair(first(last(pair)).name_to, last(last(pair))))
+    end
+    return extracted
+end
 
 function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, organ_source, variable, value)
 
@@ -684,7 +566,7 @@ function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs,
         vars_all_outputs = vars_output
 
         if haskey(timestep_mapped_outputs, proc_output)
-            vars_all_outputs = (; vars_output..., flatten_vars(timestep_mapped_outputs)...)
+            vars_all_outputs = (; vars_output..., extract_mapped_outputs(timestep_mapped_outputs[proc_output])...)
         end
 
         # If the variable is found in the outputs of the process at the other scale: