diff --git a/README.md b/README.md index 39ae36e10..58958f4ec 100644 --- a/README.md +++ b/README.md @@ -374,4 +374,4 @@ For example, PlantBiophysics.jl, which implements ecophysiological models using The package is developed so anyone can easily implement plant/crop models, use it freely and as you want thanks to its MIT license. -If you develop such tools and it is not on the list yet, please make a PR or contact me so we can add it! 😃 Make sure to read the community guidelines before in case you're not familiar with such things. +If you develop such tools and it is not on the list yet, please make a PR or contact me so we can add it! 😃 Make sure to read the community guidelines before in case you're not familiar with such things. \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index 7c3dd6a01..6890c07ef 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -42,6 +42,7 @@ makedocs(; "Implementing a model : additional notes" => "./step_by_step/implement_a_model_additional.md", ], "Execution" => "model_execution.md", + "Domain simulations" => "domain_simulation.md", "Model traits" => "model_traits.md", "AI agent skill" => "agent_skill.md", "Working with data" => [ @@ -78,6 +79,12 @@ makedocs(; "Public API" => "./API/API_public.md", "Example models" => "./API/API_examples.md", "Internal API" => "./API/API_private.md",], + "Development designs" => [ + "Multi-domain simulation design" => "./dev/multi_domain_simulation_design.md", + "Multi-domain simulation plan" => "./dev/multi_domain_simulation_plan.md", + "MAESPA-style domain example handoff" => "./dev/maespa_domain_handoff.md", + "Code cleanup audit" => "./dev/code_cleanup_audit.md", + ], "Developer guidelines" => "developers.md", "Roadmap" => "planned_features.md", ] diff --git a/docs/src/dev/code_cleanup_audit.md b/docs/src/dev/code_cleanup_audit.md new file mode 100644 index 000000000..7b9cb7794 --- /dev/null +++ b/docs/src/dev/code_cleanup_audit.md @@ -0,0 +1,90 @@ +# Code Cleanup Audit + +This page records cleanup candidates found during the multi-domain experimental +branch audit. It is intentionally biased toward code health and release-note +planning rather than immediate implementation. + +See also: + +- `release_notes_handoff.md` for the consolidated release-note source. +- `unified_scene_object_design.md` for the planned breaking replacement of the + current domain/route and multiscale-mapping split. +- `unified_scene_object_implementation_plan.md` for the implementation handoff + of that future design. + +Priority meanings: + +- P0: architectural compatibility removal or high-impact breaking cleanup. +- P1: should be handled before stabilizing the new API. +- P2: useful cleanup with moderate risk or blast radius. +- P3: lower-risk cleanup or follow-up once nearby code is touched. + +## Functions With Mergeable Intent + +These functions are not always wrong as separate Julia methods. The cleanup +target is duplicated control flow, not legitimate multiple dispatch. + +| Priority | Functions | Evidence | Recommended cleanup | +| --- | --- | --- | --- | +| Done | `_resolve_input_windowed`, `_resolve_input_interpolate`, `_resolve_input_holdlast` | `src/time/runtime/input_resolution.jl` | Policy-specific wrappers now delegate to `_resolve_input_from_policy!`, with shared vector/scalar source lookup and policy-specific sampling dispatch. | +| Done | `_normalize_meteo_reducer`, `_resolve_window_reducer` | `src/time/runtime/meteo_sampling.jl`; `src/time/runtime/input_resolution.jl` | Merged through `_normalize_time_reducer(...; context=...)`. | +| Done | `validate_meteo_inputs(model_specs, meteo)` and `validate_meteo_inputs(model_specs, backend)` | `src/time/runtime/meteo_sampling.jl`; `src/time/runtime/environment_backends.jl` | Shared missing-row collection and diagnostic formatting through `_collect_missing_meteo_rows` and `_error_missing_meteo_inputs`. | +| Done | `_required_horizon_for_export_policy` and `_required_horizon_for_policy` | `src/time/runtime/output_export.jl`; `src/time/runtime/publishers.jl` | Export code now delegates to `_required_horizon_for_policy(policy, clock.dt, source_dt)`. | +| Done | `_normalize_meteo_window` and `_runtime_meteo_window` | `src/mtg/ModelSpec.jl`; `src/time/runtime/meteo_sampling.jl` | Runtime meteo-window normalization now calls `_normalize_meteo_window`. | +| Done | Domain environment helpers for single-status and graph domains | `src/domains/domain_simulation.jl` | Shared `EnvironmentSupport` construction, environment sampling, and scatter flow while keeping thin single-status and graph-domain entry points. | +| Done | `_should_visit_domain_node` and `_should_publish_domain_key` | `src/domains/domain_simulation.jl` | Extracted `_phase_allows_hard_parent` for the shared hard-domain phase rule. | +| Done | `Status` and `StatusView` Base interface methods | `src/component_models/Status.jl`; `src/component_models/StatusView.jl` | Shared small private helpers for values, tuple conversion, iteration, and indexed iteration while keeping storage-specific access and mutation methods separate. | +| Done | Tutorial helper functions repeated in toy multiscale examples | `examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl`; `examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl` | Extracted shared MTG navigation helpers to `examples/ToyMultiScalePlantTutorial/ToyPlantHelpers.jl` and included it with `@__DIR__` so tutorial scripts remain standalone. | +| Done | Test helper flows that run a graph simulation and compare outputs | `test/helper-functions.jl` | Shared `run_graphsim_for_comparison` for generated multiscale comparison and filtered-output comparison paths. | + +## Backward Compatibility To Remove + +This section is the release-note source list. Removing these items is breaking, +and some are still internal dependencies rather than shallow exported shims. + +| Priority | Compatibility surface | Evidence | Migration note | +| --- | --- | --- | --- | +| Done | `ModelList` public API and legacy backing type | Formerly in `src/PlantSimEngine.jl`; `src/component_models/ModelList.jl`; `src/mtg/mapping/mapping.jl` | `ModelList` has been removed. Use `ModelMapping(model...; status=..., type_promotion=...)` for single-scale simulations. | +| Done | `run!(::ModelList, ...)` | Formerly in `src/run.jl` direct `ModelList` methods | `run!(::ModelList, ...)` has been removed. Wrap models in `ModelMapping` before running. | +| Done | Batch `run!` for collections of `ModelList` or single-scale mappings | Formerly in `src/run.jl` collection methods | Batch `run!([mapping1, mapping2], meteo)` and `run!(Dict(...), meteo)` are removed. Use an explicit loop or comprehension and call `run!` per mapping. | +| Done | `run!(mtg, mapping::AbstractDict, ...)` | Formerly in `src/run.jl` | Passing a raw `Dict` to multiscale `run!` is removed. Construct `ModelMapping(dict)` first, or use `ModelMapping(:Scale => models, ...)`. | +| Done | String scale names | `src/mtg/mapping/mapping.jl`; `src/mtg/MultiScaleModel.jl`; `src/mtg/model_spec_inference.jl`; `src/time/runtime/bindings.jl` | String scale names are removed. Use symbols everywhere, for example `:Leaf` instead of `"Leaf"`. | +| Done | `ModelMapping(Float64 => Float32)` as old type-promotion shorthand | Formerly in `src/mtg/mapping/mapping.jl` | `ModelMapping(Float64 => Float32)` is removed. Use `Dict(Float64 => Float32)` as the `type_promotion` value. | +| Done | Old output indexing helpers on multiscale output dictionaries | Formerly in `src/mtg/GraphSimulation.jl` | `outputs(out_dict, key)` and `outputs(out_dict, i)` are removed. Use `convert_outputs(out_dict, sink)` and index the converted table or dictionary explicitly. | + +`ModelMapping{SingleScale}` now uses an internal single-scale backing container +instead of the removed public `ModelList` API. + +## Non-Idiomatic Julia Patterns + +| Priority | Pattern | Evidence | Recommended cleanup | +| --- | --- | --- | --- | +| Done | Source-side `@assert` used for user/data validation | Formerly in MTG initialization, mapping, output conversion, and save-result helpers | Converted to explicit `if` checks and `error` messages in this cleanup pass. Remaining `@assert` uses are limited to tests and documentation examples. | +| Done | `ModelSpec(model::AbstractModel)` checks `model isa MultiScaleModel`, which is effectively dead | `src/mtg/ModelSpec.jl` | Added an explicit `ModelSpec(::MultiScaleModel)` method and removed the dead branch from the `AbstractModel` constructor. | +| Done | `Symbol("")` sentinel for same-scale/no-op mappings | `src/mtg/MultiScaleModel.jl`; `src/mtg/mapping/mapping.jl`; `src/dependencies/dependency_graph.jl` | Replaced the magic sentinel with the typed `SameScale()` marker and reject `Symbol("")` in new mappings. | +| Done | Domain selector detection by type name | `src/dependencies/hard_dependencies.jl`; `src/domains/domain_simulation.jl` | Added `AbstractDomainDependencySelector`; `AllDomains` and `HardDomains` subtype it, and detection now uses the marker type instead of type-name reflection. | +| Done | Policy handling by large `isa` branch chains | `src/time/runtime/input_resolution.jl` | Input resolution now dispatches on policy type through `_resolved_policy_value_for_source` and `_resolve_input_for_policy!`, with shared source-resolution helpers. | +| Done | Scope selectors accept strings and hard-code built-in scale names | `src/time/runtime/scopes.jl`; `src/mtg/ModelSpec.jl` | Scope selectors now reject strings at construction and runtime callable results must return `ScopeId` or `Symbol`. Built-in selector symbols remain explicit and validated. | +| Done | Normalizer fallbacks return unknown values unchanged | `src/mtg/ModelSpec.jl` | Fallbacks for input bindings, meteo bindings, and output routing now fail at construction with explicit errors. String scope selectors now error instead of being converted. | +| Done | Broad `Any` and anonymous named tuples in runtime storage | `src/mtg/mapping/mapping.jl`; `src/mtg/GraphSimulation.jl`; `src/time/multirate.jl`; `src/time/runtime/output_export.jl` | Added semantic aliases for model-rate declarations and temporal streams, typed reverse multiscale mappings as `Symbol => Symbol`, and replaced anonymous export-plan named tuples with `OutputExportPlan`. Remaining `Any` storage is for user-provided values/statuses and open extension hooks. | +| Done | Awkward container signatures with broad `AbstractArray` / verbose `where` clauses | `src/dataframe.jl`; `src/checks/dimensions.jl` | Simplified collection signatures to `AbstractVector`/`AbstractDict` forms without unnecessary `where` wrappers. | + +## Brittle Or Overloaded Code + +| Priority | Location | Risk | Recommended cleanup | +| --- | --- | --- | --- | +| Done | `src/dependencies/soft_dependencies.jl` hard-dependency redirection | Nested hard-dependency redirection is duplicated, walks parents with a depth cap, and can match by process without enough scale context. | Extracted shared owner-resolution helpers for nested hard dependencies, with scale-aware matching, ambiguity checks, and finalized soft-node validation. | +| Done | `src/domains/domain_simulation.jl` runner | Scheduling, environment sampling/scattering, route materialization, output publication, graph runtime lifecycle, and post-scene run loops were mixed in one large file. | Split domain routes, route runtime, environment bridge, output publication, scheduler helpers, graph-domain runner lifecycle, and shared run-loop orchestration into focused `src/domains/*` modules while keeping public run entry points in `domain_simulation.jl`. | +| Done | `src/time/runtime/input_resolution.jl` fallback resolution | Same-node, ancestor, and candidate-scan fallback can silently change behavior when topology or scope changes. | Built a shared source-status resolver that centralizes same-node, ancestor, vector, and unique-candidate fallback with scalar ambiguity validation. | +| Done | `src/mtg/initialisation.jl` `RefVector` population | Vector input order depends on MTG traversal order and can drift after growth/removal. | Added `reindex_runtime_topology!` to sort statuses by MTG node id and rebuild downstream `RefVector`s from current statuses after initialization and topology mutations. | +| Done | `src/mtg/mapping/compute_mapping.jl` and `src/mtg/mapping/mapping.jl` mapping sentinels/invariants | Magic sentinel values make mapping control flow fragile. | Same-scale mappings now use `SameScale()` instead of `Symbol("")`; explicit validation rejects the old sentinel. | +| Done | Domain run order | Domain order is mostly declaration order with `kind == :scene` last. Route constraints are validated by producer position only. | Added a stable domain DAG scheduler with declaration-order tie-breaking, explicit scene-phase edges, route producer-target edges, and cycle diagnostics. | +| Done | `src/mtg/add_organ.jl` topology mutation | Add/remove/reparent updates local status and refs, but scope-derived temporal keys and environment indexes need centralized invalidation. | Add/remove/reparent now centralize runtime topology reindexing; reparent clears temporal state for the moved subtree before rebuilding status and `RefVector` indexes. | +| Done | `examples/maespa_domain_example.jl` scene model | The example mixes solver math, hard-domain target orchestration, publication, soil feedback, and carbon updates. | Split scene energy-balance solving, leaf publication/carbon update, and soil feedback into separate helpers; added tests for selector mismatch, convergence failure, publication counts, and soil feedback. | +| Done | `src/dependencies/is_graph_cyclic.jl` cycle keys | Cycle detection keys nodes by model value and scale, which can conflate reused model objects. | Traversal now uses dependency-node identity through `IdDict`; cycle reports are converted back to `(model => scale)` for existing diagnostics. | +| Done | `src/time/runtime/bindings.jl` and input binding inference | Producer candidates can lose renamed source-variable identity. | Added `ProducerVariable(input, source)` dependency metadata for renamed multiscale producers, and candidate inference now matches on the consumer input while returning the producer source variable. | + +## Suggested Cleanup Order + +All originally listed cleanup items are now marked done. Keep this page as the +release-note source list and add new rows only for newly discovered cleanup work. diff --git a/docs/src/dev/maespa_domain_handoff.md b/docs/src/dev/maespa_domain_handoff.md new file mode 100644 index 000000000..4e27471a1 --- /dev/null +++ b/docs/src/dev/maespa_domain_handoff.md @@ -0,0 +1,127 @@ +# MAESPA-Style Domain Example Handoff + +The example in `examples/maespa_domain_example.jl` is the executable test bed +for the current hard-domain and scene microclimate prototype. It is also a good +migration target for the future unified scene/object design. + +## Current Example Shape + +- Two MTG-backed plant domains share scale names such as `:Plant` and `:Leaf`. +- Each plant domain uses copied PlantBiophysics subsample models: + `Monteith` for `:energy_balance`, `Fvcb` for `:photosynthesis`, and `Tuzet` + for `:stomatal_conductance`. +- `LeafState` owns `leaf_area` and `leaf_carbon`, because those are plant + bookkeeping variables and not PlantBiophysics model outputs. +- `LAIModel` runs in the scene domain and now receives leaf areas through + `ModelSpec(...) |> Inputs(...)`, which is bridged internally to the current + route materialization runtime. +- `SceneEB` runs hourly and now uses `ModelSpec(...) |> Calls(...)` to + manually run leaf `:energy_balance` targets and the shared soil + `:soil_water` target. +- Plant allocation runs daily through the normal plant-local dependency graph. + +## Manual Call Expectations + +- `Calls(:energy_balance => Many(kind=:plant, scale=:Leaf, process=:energy_balance))` + selects one target per matching leaf status through the current hard-domain + bridge. +- `Calls(:soil => One(kind=:soil, process=:soil_water))` selects the shared + soil model through the current hard-domain bridge. +- `dependency_targets(extra, :energy_balance)` returns executable leaf targets. +- `run_call!(target; publish=false)` is used during trial iterations. +- `run_call!(target; publish=true)` is used for the accepted final solution + so outputs are appended once to domain streams and `DomainSimulation.outputs`. +- Trial target runs mutate target status. Irreversible accumulators such as + `leaf_carbon` are updated only after the accepted solution. + +## MAESPA-Style Canopy Microclimate + +Input meteorology is treated as above-canopy forcing: + +- `meteo.T` is above-canopy air temperature. +- `meteo.VPD` is above-canopy VPD. +- `meteo.Wind`, `meteo.P`, and radiation variables are above-canopy drivers. + +Scene status stores below-canopy microclimate: + +- `canopy_tair` +- `canopy_vpd` +- `canopy_rh` +- `canopy_htot` +- `canopy_gcanop` + +The helper `tvpdcanopcalc(...)` ports the MAESPA-style canopy T/VPD update, and +`gbcanms(...)` ports the canopy aerodynamic conductance shape. The example uses +PlantMeteo kPa conventions and clipping equivalent to MAESPA's Pa clipping. + +`SceneEB` computes total leaf fluxes per ground area and uses those values for +the canopy microclimate update. Total leaf area is routed to `LAIModel`, which +computes `leaf_area` and `lai` in the scene domain. + +## Verification Expectations + +The focused test `test/test-maespa-domain-example.jl` should verify: + +- Species A has two leaves and species B has three leaves. +- `:energy_balance` hard-domain outputs are published once per leaf per hour. +- Soil `psi_soil` is updated through the scene hard target. +- `canopy_tair` and `canopy_vpd` remain finite and within MAESPA clipping + bounds relative to the above-canopy meteo. +- `canopy_rh` remains between 0 and 1. +- `LAIModel` sees every leaf area through the route and computes scene LAI. +- Daily allocation differs between the two plant species because their + allocation parameters differ. + +## Future Migration Target + +The MAESPA example has already moved away from explicit user-level +`Route(...)` and `HardDomains(...)`: leaf area materialization is declared with +`Inputs(...)`, and manual energy-balance calls are declared with `Calls(...)`. + +Expected migration: + +```julia +ModelSpec(LAIModel(ground_area)) |> + AppliesTo(One(scale=:Scene)) |> + Inputs(:leaf_areas => Many(kind=:plant, scale=:Leaf, process=:leaf_state, var=:leaf_area)) + +ModelSpec(SceneEB()) |> + AppliesTo(One(scale=:Scene)) |> + Calls(:leaf_energy => Many(kind=:plant, scale=:Leaf, process=:energy_balance)) |> + Calls(:soil => One(kind=:soil, process=:soil_water)) +``` + +Plant-local allocation should similarly move from `MultiScaleModel(...)` to a +scope-relative input: + +```julia +ModelSpec(AllocA(...)) |> + AppliesTo(Many(kind=:plant, scale=:Plant)) |> + Inputs(:leaf_carbon => Many(scale=:Leaf, within=Self(), var=:leaf_carbon)) +``` + +If this plant-local dependency is the normal behavior of the allocation model, +the model can provide it as a default trait instead: + +```julia +dep(::AllocA) = ( + leaf_carbon = Input(Many(scale=:Leaf, within=Self(), var=:leaf_carbon)), +) +``` + +The same applies to manual calls. A leaf energy-balance model can use `dep` to +declare its usual stomatal-conductance call, while the scene `ModelSpec` can +still override or specialize the call selection with `Calls(...)` when the +model is assembled into a MAESPA-style scene. + +Here `Self()` means the current model application object or scope. Because +`AllocA` runs at the plant scale, this selects leaves inside the current plant. +For a model running below the plant scale, use `SelfPlant()` or +`Ancestor(scale=:Plant)` when the intended scope is the containing plant. + +The migration target should treat `SceneEB`, `LAIModel`, and plant allocation +as model applications. `AppliesTo(...)` declares where each application runs; +`Inputs(...)` provides values scheduled by the runtime; `Calls(...)` provides +manual call handles for the iterative scene energy-balance solver. This keeps +the MAESPA example aligned with the unified design and avoids recreating a +separate domain-specific path. diff --git a/docs/src/dev/multi_domain_simulation_design.md b/docs/src/dev/multi_domain_simulation_design.md new file mode 100644 index 000000000..184fbd7d3 --- /dev/null +++ b/docs/src/dev/multi_domain_simulation_design.md @@ -0,0 +1,506 @@ +# Multi-Domain Simulation Design + +!!! warning "Current prototype, not the target breaking design" + This page documents the current multi-domain prototype built around + `Domain`, `SimulationMapping`, `Route`, `AllDomains`, and `HardDomains`. + The target breaking redesign is now documented in + `unified_scene_object_design.md`. New architecture work should use the + unified scene/object design as the source of truth. In that redesign, + `dep(model)` is kept as the model-level default dependency trait: current + `AllDomains(...)` value dependencies migrate toward `Input(...)` defaults, + current `HardDomains(...)` manual dependencies migrate toward `Call(...)` + defaults, and scenario-level `ModelSpec` declarations remain the final + override point. + +This page is the working design for extending PlantSimEngine from one plant or +one MTG mapping to reusable plant, soil, scene, and environment domains. + +The goal is incremental: keep existing `ModelMapping`, `ModelSpec`, +`MultiScaleModel`, hard dependencies, and multi-rate machinery, then add one +composition layer above them. + +## Goals + +- Reuse complete plant models, such as XPalm, as independent species or variety + domains. +- Assemble several plant domains with shared soil, scene, and microclimate + domains. +- Keep plant-local mappings readable and reusable. +- Make cross-domain coupling explicit. +- Support multi-rate execution from the start using `Dates.FixedPeriod` + timesteps. +- Preserve fast reference-based coupling inside existing mappings. +- Make compiled simulations inspectable by humans and agents. +- Allow exceptional same-scale duplicate writers through scenario-level + `Updates(...)` declarations. +- Let meteorology be either constant/table-driven or provided by an external + microclimate backend. + +## Non-Goals For The First Implementation + +- Do not infer cross-domain dependencies from matching variable names. +- Do not implement octree or voxel microclimate in PlantSimEngine. +- Do not solve arbitrary dynamic-topology MTG registration in the first slice; + organ addition, terminal-organ removal, recursive subtree removal, and + same-simulation reparenting are covered. +- Do not rewrite model kernels or force model authors to adopt a new `run!` + signature. + +## Domain + +A `Domain` wraps an existing mapping and gives it an identity: + +```julia +oil_palm = Domain(:oil_palm, kind=:plant, mapping=xpalm) +maize = Domain(:maize, kind=:plant, mapping=maize) +soil = Domain(:soil, kind=:soil, mapping=soil_mapping) +scene = Domain(:scene, kind=:scene, mapping=scene_mapping) +microclimate = Domain(:microclimate, kind=:environment, mapping=microclimate_mapping) +``` + +Model identity inside the compiled simulation is: + +```julia +(domain, scale, process) +``` + +not only: + +```julia +(scale, process) +``` + +This avoids renaming `:Leaf` into species-specific scale names. XPalm can keep +using `:Plant`, `:Leaf`, `:Internode`, and so on. + +For MTG runs, domains will also need selectors: + +```julia +Domain( + :oil_palm, + kind=:plant, + mapping=xpalm, + selector=node -> node[:species] == :oil_palm, +) +``` + +The selector decides which MTG nodes belong to the domain-local view. + +## SimulationMapping + +`SimulationMapping` assembles domains: + +```julia +simulation_mapping = SimulationMapping(oil_palm, maize, soil, scene) +``` + +It should preserve both domain-local and global scale views: + +```julia +status(sim, :oil_palm, :Leaf) # oil palm leaves +status(sim, :maize, :Leaf) # maize leaves +status(sim, :Leaf) # all leaves across plant domains +status(sim, :soil, :Soil) # shared soil statuses +``` + +When one domain selector matches several subtree roots, the domain-local status +view is flattened across those roots: + +```julia +forest = Domain(:forest, kind=:plant, mapping=leaf_mapping, selector=:Plant) +status(sim, :forest, :Leaf) # leaves from every selected Plant subtree +``` + +The initial `run!(mapping::SimulationMapping, meteo)` path supports +single-status domains. The `run!(mtg, mapping::SimulationMapping, meteo)` path +also supports MTG-backed domains when a domain selector resolves to one or more +subtree roots. That path reuses the existing `GraphSimulation` engine for each +selected root, advances domains one base timestep at a time, then publishes +aggregated graph outputs into the domain stream layer so single-status scene +domains can consume them during the same timestep. It is still acyclic by +domain order: domains whose `kind` is not `:scene` run first, then `:scene` +domains run. +Routes into graph domains are supported when their source domain has already +run in the current timestep. Runtime status counts are inspectable with +`explain_domain_statuses(sim)`. + +## Cross-Domain Routes + +Local variable inference remains inside one domain. Cross-domain exchange is +explicit: + +```julia +Route( + from = AllDomains(kind=:plant, scale=:Leaf, var=:Tleaf), + to = DomainRouteTarget(:scene, scale=:Scene, var=:leaf_temperatures), + policy = Integrate(), +) +``` + +Route cardinality should be explicit where needed: + +```julia +ManyToOneVector() +ManyToOneAggregate(sum) +OneToManyBroadcast() +SpatialSample() +SpatialScatterAdd() +``` + +This prevents ambiguous behavior when several plant domains publish the same +scale, process, or variable. + +In the single-status runner, routes are executable for target +`scale=:Default`: + +```julia +Route( + from = AllDomains( + kind=:plant, + process=:plant_transpiration, + var=:transpiration, + ), + to = DomainRouteTarget( + :scene, + var=:plant_transpirations, + process=:scene_evapotranspiration, + ), + cardinality = ManyToOneVector(), +) +``` + +The target variable must already exist in the target domain status. If the +target process is omitted, PlantSimEngine tries to infer a unique model at the +target domain/scale that consumes the routed variable; otherwise the route clock +is hourly/base-step. `ManyToOneVector()` materializes one value per producer. +`ManyToOneAggregate(f)` reduces producer values with `f`. Spatial and broadcast +cardinalities are defined as public route types but are reserved for the +MTG/spatial runner. + +In the current single-status runner, initialize routed vector targets with a +typed placeholder such as `[0.0]` rather than an empty vector. The route +overwrites the value before the target model runs, but the underlying +single-scale `ModelMapping` still treats an empty vector status value as +not initialized. + +The MTG runner also supports one graph-target route form: +`OneToManyBroadcast()` into an MTG-backed domain. The route writes the resolved +source value into every status at the target scale before that graph domain +runs for the current timestep. On the first timestep, the runner also seeds the +selected MTG nodes with the routed attribute before `GraphSimulation` +initialization so existing graph initialization checks still apply. This is +useful for same-timestep coupling from an earlier domain, such as a shared soil +signal broadcast to leaves. Scene-to-plant feedback remains out of scope for +this acyclic order because scene domains run after plant domains. + +## Domain-Aware Value Dependencies + +Scene and environment models often need to consume outputs from all plant +domains. Stream/value dependencies use `AllDomains(...)` and should be explicit: + +```julia +PlantSimEngine.dep(::SceneEvapotranspirationModel) = ( + plant_transpiration = AllDomains( + kind=:plant, + process=:plant_transpiration, + var=:transpiration, + policy=Integrate(), + ), + soil_evaporation = AllDomains( + kind=:soil, + process=:soil_evaporation, + var=:evaporation, + policy=Integrate(), + ), +) +``` + +The runtime resolves this to concrete `(domain, scale, process)` producers. +Scene models consume resolved values, not manually index a single +`extra.models[:Leaf]`. + +Unmatched `AllDomains(...)` selectors are reported with the consumer +`domain/scale/process`, the selector fields, available producer outputs, and +near matches when only `var` is wrong. This is part of the agent-facing API: +errors should contain enough concrete symbols for a user or AI system to repair +the mapping without reverse-engineering the compiled graph. + +In the first executable slice, scene models can read producer values with: + +```julia +plant_values = dependency_values(extra, :plant_transpiration, :transpiration) +soil_values = dependency_values(extra, :soil_evaporation, :evaporation) +``` + +When the selector declares `var`, model code can use the shorter form: + +```julia +plant_values = dependency_values(extra, :plant_transpiration) +``` + +For MTG-backed producer domains, each resolved `(domain, scale, process)` +producer can represent several node statuses. The default +`dependency_values(...)` result therefore contains one value per producer, with +graph-domain producers unwrapped as vectors of node values. Scene models that +want one flat list across all selected domains can request: + +```julia +leaf_values = dependency_values(extra, :leaf_fluxes; flatten=true) +``` + +When graph-domain producer values are reduced over a multi-rate window, the +runtime carries MTG node ids alongside the values and aggregates by node id. +This makes growth and pruning inside the window well-defined: a newly appeared +organ contributes from the timestep where it exists, and a removed organ keeps +its already-published contribution without requiring all value vectors in the +window to have the same length. + +If a selected producer is nested as a hard dependency inside a domain model, its +outputs are published when the owning parent model runs. This keeps the model +visible to scene-level `AllDomains(...)` dependencies without making the hard +dependency independently scheduled. + +## Hard Cross-Domain Dependencies + +Some coupled models need true call-stack control. Energy balance is the typical +case: a scene-scale solver may need to run leaf-scale energy, stomatal +conductance, and photosynthesis models repeatedly until leaf temperatures +converge. That is not an `AllDomains(...)` value dependency. + +Hard cross-domain dependencies use `HardDomains(...)`: + +```julia +PlantSimEngine.dep(::SceneEnergyBalanceModel) = ( + leaf_energy = HardDomains( + kind=:plant, + scale=:Leaf, + process=:leaf_energy_balance, + ), +) +``` + +Inside `run!`, the parent requests targets and runs them manually: + +```julia +function PlantSimEngine.run!(::SceneEnergyBalanceModel, models, status, meteo, constants=nothing, extra=nothing) + leaf_targets = dependency_targets(extra, :leaf_energy) + for iteration in 1:max_iterations + for target in leaf_targets + run_target!(target) + end + converged(status) && break + end + return nothing +end +``` + +Each target selects one model on one runtime status. In single-status +domains, a producer usually gives one target. In MTG-backed domains, a producer +at `scale=:Leaf` gives one target per matching leaf status. The call mutates the +status, just like a normal hard dependency call. By default it does not publish +to domain streams or outputs, which keeps trial iterations from becoming +canonical temporal outputs. The final accepted hard-dependency call can opt into +`publish=true`. + +Hard dependencies remain manual calls owned by the parent model. A hard +dependency child cannot have an independent `ModelSpec` timestep; if the child +needs its own clock, it should be coupled as a soft dependency instead. + +## Multi-Rate Semantics + +User-facing time configuration should continue to use `Dates`: + +```julia +ModelSpec(model) |> TimeStepModel(Dates.Hour(1)) +ModelSpec(model) |> TimeStepModel(Dates.Day(1)) +``` + +For the first implementation, use `Dates.FixedPeriod` only. `Dates.Month` and +`Dates.Year` are calendar-dependent and should error unless a calendar-aware +runtime is added. + +Cross-domain dependencies and routes carry a temporal policy: + +```julia +HoldLast() +Interpolate() +Integrate() +Aggregate() +``` + +The domain key must be part of temporal state keys from the beginning to avoid +collisions between species, soil, and environment streams. + +## Variable Updates + +Duplicate writers remain errors by default. Exceptional updates are declared by +the scenario author, not the model author: + +```julia +ModelSpec(LeafPruning()) |> + Updates(:leaf_biomass; after=:carbon_allocation) +``` + +Rules: + +- one canonical producer is allowed; +- additional same-scale writers must declare `Updates(...)`; +- `after` adds an ordering edge; +- if several update writers target the same variable, they must also be ordered + relative to each other; +- if both models run at the same `DateTime`, the updater runs after the + producer; +- if only the updater runs, it updates the latest available state; +- downstream consumers read the updated value. +- in multi-rate input binding inference, an ordered update chain is treated as + one effective source by selecting the unique terminal updater for the updated + variable. + +Only variables with duplicate writers need annotation. + +## Meteorology And Microclimate + +Models should declare meteorological fields explicitly: + +```julia +meteo_inputs_(::LeafEnergyBalanceModel) = ( + T = -Inf, + Rh = -Inf, + Wind = -Inf, + Ri_PAR_f = -Inf, + CO2 = -Inf, +) + +meteo_outputs_(::MicroclimateModel) = ( + T = -Inf, + Rh = -Inf, + Wind = -Inf, + Ri_PAR_f = -Inf, + CO2 = -Inf, +) +``` + +`inputs_` and `outputs_` remain object status variables. `meteo_inputs_` and +`meteo_outputs_` describe weather/environment variables; `meteo_inputs` and +`meteo_outputs` are the public key accessors. + +PlantSimEngine should define the microclimate backend interface, not the octree +or voxel implementation: + +```julia +abstract type AbstractEnvironmentBackend end + +get_nsteps(backend) +base_step_seconds(backend) +sample(backend, variable, support, time) +sample_environment(backend, support, time, variables) +scatter!(backend, variable, support, value, time) +update_index!(backend, entities) +``` + +`GlobalConstant(meteo)` is the simple backend and preserves current behavior: +all models see the same meteo object or meteo row at a given timestep. Plain +meteo passed to `run!(SimulationMapping, meteo)` is wrapped automatically. + +Backends receive an `EnvironmentSupport(domain, scale, process, status)` so +they can decide whether a sampled value is global, plant-local, organ-local, or +spatially resolved. The same protocol is used for single-status domains and +MTG-backed domains; in graph domains, `status` is the current node status, so +the backend can inspect node geometry, scale, domain, or any status variable. +Octree, voxel, layered-canopy, and CFD backends should live in specialized +packages. + +After each domain step, the domain runtime calls: + +```julia +update_index!(backend, entities) +``` + +where `entities` is a small collection of `(domain, kind, scale, statuses, +state)` rows. Spatial backends can use this hook to refresh their entity index +after geometry changes, growth, pruning, or status updates. `GlobalConstant` +ignores the hook. + +When a model declares `meteo_outputs_`, the domain runtime scatters those values +to the backend after the model runs: + +```julia +meteo_outputs_(::MicroclimateUpdateModel) = (T = -Inf,) +outputs_(::MicroclimateUpdateModel) = (T = -Inf,) +``` + +The same variable must currently be available on the model status, usually by +declaring it in `outputs_`. This keeps the existing `run!` signature unchanged +while giving backends a clear `scatter!` hook. In MTG-backed domains, the +scatter hook is called once per node status after the model runs. `GlobalConstant` +is immutable and errors if a model tries to scatter into it. + +## Growth And Dynamic Entities + +Dynamic organ creation must go through a central registration API. When a new +organ is added, the runtime must update: + +- domain-local status views; +- global scale views; +- cross-domain routes; +- outputs; +- temporal buffers; +- environment spatial indexes. + +This should not be scattered through model code. In MTG-backed domains, models +should keep using the existing: + +```julia +add_organ!(parent_node, graph_simulation, link, symbol, scale; kwargs...) +``` + +from their `run!` implementation. The domain runner passes the underlying +`GraphSimulation` as `extra`, so existing growth models can register new organs +without knowing they are part of a `SimulationMapping`. + +For the current domain runner: + +- `add_organ!` initializes the new node status and updates multiscale + `RefVector` wiring through the existing MTG initialization helpers. +- `remove_organ!` supports terminal MTG nodes by default and internal-node + subtree deletion with `recursive=true`. It removes node statuses, detaches + references from downstream `RefVector`s, clears node-scoped temporal cache and + stream entries, and deletes the MTG nodes. +- `reparent_organ!` moves an already simulated node under another already + simulated parent in the same `GraphSimulation`, validating that both nodes are + registered and that the move does not create a cycle. Statuses and + `RefVector`s continue to reference the same node objects, so no status + rewiring is needed for this case. +- `status(sim, domain, scale)` and `status(sim, scale)` read the live + `GraphSimulation` status vectors, so newly added organs are visible after + registration. If a declared graph scale currently has no active statuses, + these helpers return `Status[]` rather than throwing a dictionary lookup + error, and `explain_domain_statuses(sim)` reports the scale with + `nstatuses=0`. +- graph-domain output publication runs after each graph-domain timestep and + includes newly registered statuses while skipping removed terminal organs. +- `update_index!(backend, entities)` is called after each domain step so + external microclimate backends can refresh their spatial/entity index. + +The current tests cover a new graph-domain leaf publishing an hourly stream +that is consumed by a daily model with `Integrate()`, online +`OutputRequest(...)` exports from dynamically created leaves, several leaves +created in the same timestep, custom callable scopes returning `ScopeId`, +terminal leaf removal in direct and multirate graph-domain coupling, repeated +terminal create/remove cycles, recursive internal-node subtree deletion, and +same-simulation topology reparenting. + +## Agent-Friendly Requirements + +The compiled simulation must be explainable as structured data: + +```julia +explain_domains(sim) +explain_routes(sim) +explain_schedule(sim) +explain_writers(sim) +explain_domain_dependencies(sim) +``` + +Errors should include the conflicting domains/scales/processes and a suggested +fix. For example, duplicate writers should suggest `Updates(:var; after=...)`. diff --git a/docs/src/dev/multi_domain_simulation_plan.md b/docs/src/dev/multi_domain_simulation_plan.md new file mode 100644 index 000000000..1b6f799c8 --- /dev/null +++ b/docs/src/dev/multi_domain_simulation_plan.md @@ -0,0 +1,270 @@ +# Multi-Domain Simulation Implementation Plan + +!!! warning "Current prototype plan" + This page tracks what has been implemented in the current multi-domain + prototype. The next breaking architecture plan lives in + `unified_scene_object_implementation_plan.md`. That plan keeps `dep(model)` + as the model-level default dependency trait, with future `Input(...)` and + `Call(...)` defaults overridden by scenario-level `ModelSpec` + configuration. + +This page tracks the implementation plan for the multi-domain work. The design +reference is `multi_domain_simulation_design.md`. + +## Milestone 1: Executable Single-Status Domain Slice + +Done when: + +- `Domain`, `SimulationMapping`, `DomainSimulation`, `DomainModelKey`, and + `AllDomains` exist. +- Domains can wrap existing single-scale `ModelMapping`s. +- A `SimulationMapping` can run plant and soil domains hourly and a scene + domain daily with `Dates.Hour(1)` and `Dates.Day(1)`. +- Single-status domains can contain models with different `ModelSpec` + timesteps. Implemented. +- Scene models can consume resolved cross-domain producer streams with + `dependency_values`. +- `AllDomains(...; var=:x)` can declare the consumed output variable, allowing + `dependency_values(extra, :dependency_name)` and earlier validation of + producer matches. Implemented for the single-status domain runner. +- Scene models can consume a model nested as a hard dependency inside a domain; + the nested model output is published when its owning parent runs. + Implemented for the single-status domain runner. +- `explain_domains`, `explain_schedule`, and + `explain_domain_dependencies` return structured rows. +- A test example covers two plant domains, one soil domain, and one scene + evapotranspiration model. + +Likely files: + +- `src/domains/domain_simulation.jl` +- `src/PlantSimEngine.jl` +- `test/test-domain-simulation.jl` +- `test/runtests.jl` + +## Milestone 2: Agent-Friendly Inspection And Validation + +Current status: implemented for the current domain runner surface. +`explain_domain_models(...)` returns expanded rows, the initial runner +validates unsupported cases early, mixed rates inside single-status domains are +scheduled independently, and explicit `Route(...)` materialization is +implemented for single-status domains with `ManyToOneVector()` and +`ManyToOneAggregate(...)`. + +Done when: + +- `explain_domain_models(simulation_mapping)` returns expanded domain model + rows. Implemented. +- Errors include domain, scale, process, variable, and suggested fixes. + Implemented for unmatched `AllDomains(...)` dependencies and routes, route + targets, duplicate domains, and unsupported domain/route shapes. +- Duplicate domain names error early. Implemented. +- Mixed timesteps inside one initial domain are scheduled independently. + Implemented for single-status domains. +- `AllDomains(...)` selectors report unmatched dependencies with context. + Implemented. +- `Route(...)` materializes cross-domain producer streams into target domain + status variables. Implemented for single-status domains. +- `explain_domain_dependencies(simulation)` reports resolved producers, + temporal policy, and the declared dependency variable. Implemented. +- `explain_routes(simulation)` reports resolved route producers, target + variables, cardinality, temporal policy, and effective route clocks. + Implemented. + +Likely tests: + +- duplicate domain name; +- unmatched `AllDomains`; +- scene domain with multiple scene processes; +- domain with mixed model rates in the initial runner. +- scene dependency targeting a hard-dependency child inside a plant domain. +- vector and aggregate cross-domain routes. + +## Milestone 3: `Updates(...)` In `ModelSpec` + +Current status: implemented for same-scale dependency graphs, single-scale +runs, and MTG-backed domain runs. Downstream input binding inference treats an +unambiguous ordered update chain as one effective producer by selecting the +terminal updater. + +Done when: + +- `Updates(:var; after=:process)` exists as a scenario-level annotation. + Implemented. +- Duplicate writers remain errors unless additional writers declare updates. + Implemented for canonical same-scale writers. +- `after` creates an ordering edge. + Implemented for executable soft dependency nodes. +- Downstream consumers read the terminal updated value when the update order is + unambiguous. Implemented for MTG-backed domains. +- Errors suggest the exact `Updates(:var; after=...)` fix. + Implemented for duplicate writers and invalid update declarations. +- Only variables with duplicate writers need annotation. + Implemented. + +Likely files: + +- `src/mtg/ModelSpec.jl` +- dependency graph construction files +- model spec validation files +- tests for duplicate writers and ordered updates + +## Milestone 4: Meteo Traits + +Current status: trait surface and runtime field validation are implemented. +Environment backend coupling is implemented for the single-status domain +runner through the protocol described in Milestone 6. + +Done when: + +- `meteo_inputs(model)` and `meteo_outputs(model)` default to empty keys from + `NamedTuple()`. Implemented. +- Existing meteo validation can check required fields when traits are present. + Implemented for constant/table meteo field presence, including + `MeteoBindings(source=...)` remapping and public + `validate_meteo_inputs(mapping, meteo)` checks. +- `explain_model_specs` or a new explanation helper reports meteo needs. + Implemented in `explain_model_specs` and `explain_domain_models`. +- Trait declarations support simple NamedTuple defaults first. + Implemented. + +Likely files: + +- `src/processes/models_inputs_outputs.jl` +- `src/time/runtime/meteo_sampling.jl` +- documentation for model authors + +## Milestone 5: MTG-Backed Domains + +Current status: timestep-interleaved MTG-backed domain execution is implemented +for domain selectors that resolve to one or more subtree roots. The runner +advances non-scene domains, including graph domains backed by the existing +`GraphSimulation`, then advances single-status scene domains in the same base +timestep. Graph-domain outputs are aggregated per domain and published into +per-scale streams for `Route(...)` and `AllDomains(...)` consumers. + +Done when: + +- `Domain(..., selector=...)` partitions MTG nodes into domain-local views. + Implemented for one or more selected subtree roots per graph domain. +- Domain-local statuses and global scale statuses are both available. + Domain-local statuses are available through `status(sim, domain, scale)`. + Global cross-domain views are available through `status(sim, scale)` when no domain + has the same name, and runtime counts are reported by + `explain_domain_statuses(sim)`. +- Cross-domain routes can consume all statuses from selected domains. + Implemented for graph-domain sources routed into single-status + domains. Also implemented for `OneToManyBroadcast()` routes into graph + domains when the source domain runs earlier in the current timestep. +- Existing single-domain MTG behavior still works. + Reused by the domain runner through `GraphSimulation`. + +Likely files: + +- `src/mtg/initialisation.jl` +- `src/mtg/GraphSimulation.jl` +- new domain graph simulation code +- tests with two plant species sharing `:Leaf` scale names + +## Milestone 6: Environment Provider Interface + +Current status: protocol and constant backend are implemented for the domain +runner. Custom backends are supported by single-status domains and MTG-backed +domains. Spatial backends remain external-package work. + +Done when: + +- PlantSimEngine defines the backend protocol for sampling and scattering + meteo/environment variables. Implemented with `AbstractEnvironmentBackend`, + `EnvironmentSupport`, `sample`, `sample_environment`, `scatter!`, + `update_index!`, `get_nsteps`, and `base_step_seconds`. +- A constant meteo backend supports current behavior. Implemented with + `GlobalConstant`, and plain meteo passed to `run!(SimulationMapping, meteo)` + is wrapped automatically. +- External packages can implement spatial backends without PlantSimEngine + knowing whether they use voxels, octrees, layers, or another structure. + Implemented for the protocol surface; full spatial examples remain future. +- Environment bindings are visible to explanation helpers. Implemented with + `explain_environment(simulation)` plus `meteo_inputs_` in + `explain_domain_models`. +- Domain models declaring `meteo_outputs_` scatter same-named status values into + mutable backends after `run!`. Implemented for the single-status domain + runner and MTG-backed domain runner; `GlobalConstant` errors because it is + immutable. +- Domain steps call `update_index!(backend, entities)` after model execution so + mutable/spatial backends can refresh their entity indexes. Implemented for + single-status domains and MTG-backed domains; `GlobalConstant` is a no-op. + +Likely API: + +```julia +sample(backend, variable, support, time) +scatter!(backend, variable, support, value, time) +update_index!(backend, entities) +``` + +## Milestone 7: Growth Registration + +Current status: implemented for MTG-backed domains that use the existing +`add_organ!` API inside model `run!` methods, and for terminal or recursive +subtree removal with `remove_organ!`, and for same-simulation topology +reparenting with `reparent_organ!`. Domain status views and graph-domain stream +publication observe the live `GraphSimulation` status vectors, dynamic producer +streams are extended lazily, removed organs are detached from downstream +`RefVector`s and temporal caches, reparented organs keep their existing status +and reference identity, and environment backends receive `update_index!` calls +after each domain step. + +Done when: + +- new organs are added through one runtime API. Implemented by reusing + `add_organ!` from graph-domain models. +- domain-local and global status views are updated. Implemented because + `DomainSimulation` reads the live graph-domain status vectors. +- graph-domain output publication includes newly registered statuses. + Implemented in the timestep-interleaved domain runner. +- environment indexes can be updated by the backend. Implemented through + `update_index!(backend, entities)` after each domain step. +- organs can be removed through one runtime API. Implemented with + `remove_organ!`, which removes the node status, detaches downstream + `RefVector` references, removes node-scoped temporal cache and stream entries, + and deletes the MTG node. Terminal deletion is the default; internal-node + subtree deletion is available with `recursive=true`. +- already simulated organs can be reparented through one runtime API. + Implemented with `reparent_organ!` for moves inside the same active + `GraphSimulation`. +- output and temporal buffers are resized or lazily extended for all covered + dynamic multi-rate cases. Regular graph outputs use the existing + `save_results!` resizing path. Dynamic producer streams are covered for an + hourly producer added during the run and consumed by a daily model with + `Integrate()`. Online `OutputRequest(...)` exports are covered for a + dynamically created leaf using plant scope, several leaves created in the + same timestep, and custom callable scopes returning `ScopeId`. Terminal + removal is covered for direct `RefVector` coupling, multirate temporal + streams, repeated terminal create/remove cycles, and recursive internal-node + subtree deletion. Same-simulation reparenting is covered for topology + mutation while preserving status and reference identity. + +## Executable Examples + +The first executable domain example in `docs/src/domain_simulation.md` uses: + +- two plant domains with different parameters and several hourly models each; +- one soil domain with several hourly models; +- one scene domain with a daily evapotranspiration model; +- explicit `AllDomains(...)` dependencies from the scene model to plant + transpiration and soil evaporation; +- `Dates.Hour(1)` for plant/soil domains and `Dates.Day(1)` for the scene + model. + +The MAESPA-style example in `examples/maespa_domain_example.jl` exercises the +hard-dependency path: + +- two MTG-backed plant domains with different models and parameters; +- one shared soil domain; +- one scene energy-balance model using `HardDomains(...)`; +- manual calls through `dependency_targets(...)` and `run_target!(...)`; +- trial iterations with `publish=false`, followed by final accepted calls with + `publish=true`; +- hourly scene/plant/soil energy-balance models and daily plant allocation + models. diff --git a/docs/src/dev/release_notes_handoff.md b/docs/src/dev/release_notes_handoff.md new file mode 100644 index 000000000..98f624cdd --- /dev/null +++ b/docs/src/dev/release_notes_handoff.md @@ -0,0 +1,241 @@ +# Release Notes Handoff + +This page is the persistent release-note source for work done during the +multi-domain and cleanup branch. Keep it factual: mark what is implemented, +what is removed, and what is only planned. + +## Implemented Breaking Cleanup + +Source details live in `code_cleanup_audit.md`. + +- Removed public `ModelList` usage. Use `ModelMapping(model...; status=...)` + for single-scale simulations. +- Removed direct `run!(::ModelList, ...)`; wrap models in `ModelMapping`. +- Removed batch `run!` over collections/dictionaries of single-scale mappings; + use explicit loops or comprehensions. +- Removed raw `Dict` multiscale `run!(mtg, dict, ...)`; construct + `ModelMapping(dict)` first. +- Removed string scale names. Use symbols, for example `:Leaf`. +- Removed `ModelMapping(Float64 => Float32)` promotion shorthand. Use a + `Dict(Float64 => Float32)` as the `type_promotion` value. +- Removed old multiscale output indexing helpers. Convert outputs explicitly + before indexing. +- Replaced the `Symbol("")` same-scale sentinel with `SameScale()`. +- Replaced many source-side validation `@assert`s with explicit errors. +- Added `Updates(:var; after=:process)` for ordered duplicate writers. + +## Implemented Multi-Domain / Scene Prototype Features + +These features exist in the current prototype, but may be replaced by the +unified scene/object API in a future breaking pass. + +- `Domain`, `SimulationMapping`, `DomainSimulation`, and `DomainModelKey`. +- Single-status and MTG-backed domains. +- Domain selectors that can select one or more MTG subtree roots. +- Domain-local and global status views: + `status(sim, :plant_A, :Leaf)` and `status(sim, :Leaf)`. +- Multi-rate domain execution using `Dates` periods. +- `AllDomains(...)` stream/value dependencies. +- `Route(...)` materialization with `ManyToOneVector`, + `ManyToOneAggregate`, and limited `OneToManyBroadcast` graph support. +- `HardDomains(...)`, `dependency_targets`, `ModelTarget`, and + `run_target!` for manually controlled hard-domain calls. +- Environment backend protocol: + `AbstractEnvironmentBackend`, `GlobalConstant`, `EnvironmentSupport`, + `sample_environment`, `scatter!`, `update_index!`, `get_nsteps`, and + `base_step_seconds`. +- `meteo_inputs_` and `meteo_outputs_` declarations and validation. +- Dynamic MTG add/remove/reparent runtime reindexing. +- Domain explanation helpers: + `explain_domains`, `explain_domain_models`, `explain_domain_statuses`, + `explain_schedule`, `explain_domain_dependencies`, and `explain_routes`. + +## Implemented MAESPA-Style Example Changes + +The current `examples/maespa_domain_example.jl` is the main executable example +for multi-plant scene coupling. + +- Uses copied PlantBiophysics subsample models: + `Monteith`, `Fvcb`, and `Tuzet`. +- Uses two MTG-backed plant domains with different parameters and shared scale + names such as `:Plant` and `:Leaf`. +- Uses a shared soil model. +- Uses `SceneEB` with `ModelSpec(...) |> Calls(...)` to manually run leaf + `:energy_balance` and soil `:soil_water` targets through the current + hard-domain bridge. +- Ports MAESPA-style canopy air temperature and VPD update through + `tvpdcanopcalc` and `gbcanms`. +- Treats input meteorology as above-canopy forcing and writes below-canopy + microclimate to scene status fields: + `canopy_tair`, `canopy_vpd`, `canopy_rh`, `canopy_htot`, and + `canopy_gcanop`. +- Adds `LAIModel` and declares plant leaf-area materialization with + `ModelSpec(...) |> Inputs(...)`, bridged internally to the current route + runtime. +- Computes plant allocation daily from plant-local `leaf_carbon` vectors. +- Adds `run_call!` as the unified scene/object spelling for manually executing + current `ModelTarget` call handles. +- Adds model-level `Input(...)` and `Call(...)` dependency defaults through + `dep(model)`, with scenario-level `Inputs(...)` and `Calls(...)` overriding + those defaults in `ModelSpec`. +- Adds initial registry-backed scene selector resolution with + `resolve_object_ids` and `resolve_objects` for global, self-relative, + plant-relative, ancestor-relative, and named-scope object selections. +- Adds `explain_scopes(scene)` for structured scope diagnostics. It reports + the scene scope, object subtree scopes, named `Scope(...)` entries, and + scale/kind/species label groups with concrete object ids. +- Adds the first compiled scene/object view with `compile_scene`, + `CompiledScene`, `CompiledSceneApplication`, `CompiledSceneInputBinding`, + `CompiledSceneCallBinding`, `explain_scene_applications`, + `explain_bindings`, and `explain_calls`. +- The compiled scene view resolves `AppliesTo(...)`, `Inputs(...)`, and + `Calls(...)` to object ids ahead of runtime, and reports temporal policy, + window, carrier hints, and callee application ids for agent-readable + diagnostics. +- Unscoped scene/object dependency selectors now infer scope from the consumer: + scene consumers default to `SceneScope()`, while non-scene consumers default + to `Self()`. Cross-scope shared dependencies, such as leaf models reading + soil state, should use `within=SceneScope()` explicitly. +- Adds status-backed compiled input carriers for the scene/object view: + scalar shared refs, homogeneous `RefVector`s, and `ObjectRefVector` fallback + carriers. `input_carrier`, `input_value`, and `has_reference_carrier` expose + them for tests, diagnostics, and future runtime execution. +- `explain_bindings` now reports stable carrier kind and copy/reference + semantics, making reference-wired inputs and materialized temporal values + explicit for users and agents. +- Adds scene binding cache helpers: + `refresh_bindings!`, `bindings_dirty`, `compiled_bindings`, and + `scene_revision`. Object registration, removal, and reparenting invalidate + cached compiled bindings before the next refresh. +- Adds scene/object environment binding cache helpers: + `refresh_environment_bindings!`, `compile_environment_bindings`, + `CompiledEnvironmentBinding`, `CompiledEnvironmentBindings`, + `environment_bindings_dirty`, `compiled_environment_bindings`, + `environment_revision`, and `explain_environment_bindings`. +- Adds `geometry`, `position`, and `bounds` accessors for scene objects/statuses. + Environment binding refreshes call `update_index!(backend, entities)` before + binding objects to backend cells/layers, so spatial backends can precompute + scene-wide lookup structures. +- Object movement now invalidates environment bindings without rebuilding the + structural object/model binding cache. +- Adds public geometry lifecycle helpers: + `update_geometry!(scene, object, geometry; invalidate_environment=true)` and + object-scoped `mark_environment_binding_dirty!(scene, object)`. They + currently invalidate the scene environment binding cache and leave room for + finer-grained dirty tracking later. +- Adds the first scene/object runtime with `run!(scene; steps=...)`. + It materializes compiled `Inputs(...)` carriers, samples bound environment + inputs, and executes generic model kernels on object `Status` values. +- Scene/object compiler now infers simple same-object value bindings from + `inputs_`/`outputs_` when one producer is unambiguous. `explain_bindings` + reports each binding origin, including `:declared` and + `:inferred_same_object`. +- Compiled input bindings now validate `Inputs(...)` `process=`/`application=` + filters when they are provided, and `explain_bindings` reports + `source_application_ids`, `process`, and `application`. +- `compile_scene` now errors for required `inputs_(model)` variables that are + neither bound through `Inputs(...)`/inference nor present on the target object + `Status`. +- `compile_scene` now rejects `Inputs(...)` entries whose receiving variable is + not declared by the model's `inputs_`, making binding typos explicit at + compile time. +- `compile_scene` now validates status-backed non-temporal `Inputs(...)` + source availability, so bindings that select existing source objects but no + source `Status` reference fail at compile time instead of becoming no-ops. +- Scene/object runtime now publishes model outputs to scene-local temporal + streams and resolves temporal `Inputs(...)` with `HoldLast`, `Integrate`, + and `Aggregate` policies before consumer execution. +- Scene/object runtime now scatters values declared by `meteo_outputs_(model)` + back to the bound environment backend after each model call, using the + existing `scatter_environment_outputs!` backend protocol. +- Scene/object root applications now honor `TimeStep(...)` values backed by + `Dates.Period` scheduling. `explain_schedule` reports normalized clocks and + whether an application is root-scheduled or manual-call-only. +- Scene/object execution now uses a stable topological application order + compiled from `Inputs(...)` producer edges and `Updates(...)` ordering. + Dependencies on manual-call-only applications are redirected to their parent + caller, same-timestep cycles fail during compilation, and + `explain_schedule` reports `execution_index`. +- `CompiledScene` now pre-indexes input and call bindings by application and + object id. Runtime input materialization and hard-call lookup no longer scan + all scene bindings for every object/model invocation. +- `CompiledScene` now pre-indexes applications by application id, removing + application scans from hard-call target resolution and dictionary rebuilding + from ordered execution setup. +- `CompiledEnvironmentBindings` now pre-indexes environment bindings by + application and object id, removing the scene-wide binding scan from + environment sampling and output scattering. +- Adds `SceneRunContext` and `SceneCallTarget`; scene/object models can use + `dependency_target(s)(extra, :name)` plus `run_call!` for manual + `Calls(...)` execution. +- Applications selected by `Calls(...)` are skipped by the root + `run!(scene)` loop and execute only through explicit `run_call!`, preserving + parent-controlled hard-call execution. +- Adds scene/object duplicate-writer validation in `compile_scene`. A variable + may have only one canonical writer per object unless later writers declare + `Updates(:var; after=...)`, where `after` can match a previous application + id/name or process. +- Adds `explain_writers(compiled)` to report object-variable writer groups, + duplicate writers, and the `Updates(...)` declarations that validate ordered + updates. +- Adds the first reusable object-template path with `ObjectTemplate` and + `ObjectInstance`. Templates bundle reusable `ModelSpec`s and default + `kind`/`species` labels; instances mount them inside a named object subtree. +- `Scene(...)` accepts mounted instances whose roots are either owned objects + or references to separately supplied scene objects. +- Template applications are scoped to their instance and receive stable + instance-prefixed application ids. Unmodified instances share the template's + model objects, while instance overrides can replace one application by name + or process when the replacement implements the same process. +- Adds `Override(...)` and `ObjectInstance(...; object_overrides=...)` for + exceptional organs. Overrides are resolved during compilation to concrete + object ids without splitting the logical application or changing its + dependency bindings. +- Template models, template parameter metadata, and replacement models are + retained by reference. The runtime does not copy models or mutate fields to + apply parameter overrides. +- Override validation requires the same process and declared status/environment + variable names. Application explanations report model storage, dispatch + mode, overridden object ids, and replacement model types. +- Adds `explain_instances(scene)` and instance membership in + `explain_objects(scene)`. Instance rows expose roots, current object + membership, mounted applications, overrides, template labels, and + reference-based parameter ownership. +- Objects created below a mounted instance inherit missing template `kind` and + `species` labels. Membership explanations use the current topology rather + than a copied instance object list. + +## Planned Future Breaking Redesign + +The target design is documented in: + +- `unified_scene_object_design.md` +- `unified_scene_object_implementation_plan.md` + +Expected future migration: + +- model mappings should be described as model applications: + `ModelSpec(model; name=...) |> AppliesTo(...) |> Inputs(...) |> Calls(...)`; +- `MultiScaleModel(...)` -> `Inputs(...)`. +- `Route(...)` for normal value coupling -> consumer-side `Inputs(...)`. +- `AllDomains(...)` selectors -> object selectors inside `Inputs(...)`. +- `HardDomains(...)` -> `Calls(...)`. +- `dep(model)` remains the model-level trait for default dependency intent: + defaults can become `Input(...)` value bindings or `Call(...)` manual model + calls, and scenario-level `ModelSpec` configuration overrides them. +- `Domain(...)` as user-facing assembly -> scene object templates and + instances. +- model target scales/domains -> `AppliesTo(...)` object selectors. +- `InputBindings(...)` -> source, policy, and window information on + `Inputs(...)`. +- `MeteoBindings(...)` and `MeteoWindow(...)` -> automatic environment + binding plus optional `Environment(...)` overrides. +- `OutputRouting(...)` -> model-application output policy. +- `ScopeModel(...)` -> `AppliesTo(...)` plus selector scopes. +- `PreviousTimeStep(...)` remains supported as a temporal/cycle-breaking + marker in the unified object-address graph. +- explicit per-model meteo wiring -> automatic environment resolver plus + cached environment bindings. + +Important: the future redesign is not implemented yet. Do not describe it as +released behavior until the old examples have been migrated and tests pass. diff --git a/docs/src/dev/unified_scene_object_design.md b/docs/src/dev/unified_scene_object_design.md new file mode 100644 index 000000000..e92fffa7e --- /dev/null +++ b/docs/src/dev/unified_scene_object_design.md @@ -0,0 +1,641 @@ +# Unified Scene/Object Design + +This page records the target breaking design discussed after the multi-domain +prototype. It intentionally supersedes the user-facing distinction between +`MultiScaleModel(...)` mappings and `Route(...)` cross-domain materialization. + +The central idea is: + +> Domains and scales are not fundamentally different concepts. They are both +> selections over objects in one scene. + +The engine should expose one way to say "this model input comes from these +objects" and one way to say "this model must manually call these models". The +compiler can then choose whether the runtime carrier is a `Ref`, `RefVector`, +temporal stream, route materialization, or callable model handle. + +The public API should be simple enough to remember as: + +```julia +Scene +Object +ModelSpec + +AppliesTo(...) +Inputs(...) +Calls(...) +Updates(...) +TimeStep(...) +Environment(...) +``` + +Everything else should either be a selector, a trait declared by the model +author, or an internal compiled carrier. + +## Core Concepts + +### Scene + +A `Scene` is the whole simulation universe. It contains: + +- simulated objects; +- model applications; +- environment providers; +- time/runtime state; +- caches for object selections and environment bindings. + +Plants, soil, atmosphere, microclimate grids, organs, sensors, and artificial +objects all live in the same scene-level object graph. + +### Object + +An object is any simulated entity with identity. It may have: + +- a unique object id; +- one or more labels, such as `scale=:Leaf`, `kind=:plant`, + `species=:oil_palm`; +- parent/child links; +- geometry or position; +- status variables; +- model applications. + +The engine must not prescribe a plant architecture. A plant can be described as +`Plant -> Internode -> Leaf`, `Plant -> Axis -> Segment -> Leaf`, +`Plant -> Metamer -> Organ`, or another topology. The engine only needs object +identity, labels, and relations. + +### Scale + +A scale is a label on objects, not a separate runtime layer. Examples: + +```julia +:Scene +:Plant +:Axis +:Internode +:Leaf +:Soil +:SoilLayer +:Voxel +``` + +### Scope + +A scope is a named or inferred subset of objects. Examples: + +```julia +SceneScope() +Self() +SelfPlant() +Ancestor(scale=:Plant) +Scope(:oil_palm) +Kind(:plant) +Species(:oil_palm) +``` + +`Self()` means the current model application object or scope. It does not +always mean "the current plant". If a model runs on a `:Plant`, `Self()` means +that plant object or subtree. If a model runs on an `:Axis`, it means that axis. +If a model runs on a `:Leaf`, it means that leaf. If a model runs on the scene +object, it means the scene object/scope. + +`SelfPlant()` is the nearest containing plant scope. The more generic form is +`Ancestor(scale=:Plant)`. Use these when a model running below the plant scale +must access siblings or state inside the containing plant. + +Reusable plant models should default to scope-relative queries. If an +allocation model is applied to each `:Plant`, `Many(scale=:Leaf, within=Self())` +means "the leaves inside this plant", not all leaves in the scene. The same +query applied to an axis-scale model would mean "the leaves inside this axis". + +Scene-level models widen the scope explicitly with `within=SceneScope()`. + +### Object Template And Instance + +An object template is a reusable model/parameter bundle, for example one oil +palm species model. An object instance is one concrete object in the scene. + +The same template can be mounted several times: + +```julia +oil_palm = ObjectTemplate( + kind=:plant, + species=:oil_palm, + mapping=oil_palm_mapping, + parameters=oil_palm_parameters, +) + +scene = Scene( + ObjectInstance(:palm_1, oil_palm; root=node1), + ObjectInstance(:palm_2, oil_palm; root=node2), + ObjectInstance(:palm_3, oil_palm; root=node3), + ObjectInstance(:palm_4, oil_palm; root=node4), +) +``` + +Models and parameters can be overridden at instance or object level: + +```julia +ObjectInstance(:palm_2, oil_palm; overrides=( + stomatal_conductance = Tuzet(; g1=3.2), +)) + +Override( + object=:leaf_12, + process=:photosynthesis, + model=Fvcb(; VcMaxRef=90.0), +) +``` + +Ownership is reference-based and explicit: + +- a template retains the supplied model and parameter objects without copying; +- unchanged instances share those exact objects; +- an instance override replaces one complete model application with another + user-owned model object; +- an object override replaces that application only for the selected object; +- PlantSimEngine does not mutate model fields or implicitly merge parameter + dictionaries. + +Overrides must preserve the model contract: process identity and declared +status/environment variable names cannot change. Parameter-only overrides of +the same concrete model type retain concrete runtime dispatch. Heterogeneous +alternative implementations are supported but may require dynamic dispatch for +the exceptional application. + +### Model Kernel And Model Application + +A model kernel is the reusable model implementation written by a modeler. It +defines a process, parameters, `inputs_`, `outputs_`, optional `dep` defaults, +optional environment traits, and `run!`. + +A model application is the scenario-specific use of that kernel on selected +objects, at a selected rate, with selected value inputs, model calls, update +rules, output routing, and environment binding behavior. + +The model kernel should not need to know: + +- the species it will be used with; +- the scene it will be embedded in; +- the timestep chosen by the user; +- whether its inputs come from local state, another scale, another object, a + temporal stream, units, automatic differentiation values, or uncertainty + wrappers. + +The scenario owns those decisions through `ModelSpec`. + +Target shape: + +```julia +ModelSpec(LeafEnergyBalance(); name=:leaf_energy) |> + AppliesTo(Many(kind=:plant, scale=:Leaf)) |> + Inputs(...) |> + Calls(...) |> + TimeStep(Hour(1)) +``` + +Application names are optional but important when the same process appears more +than once in the same object set. Other declarations can target either +`process=:photosynthesis` or `name=:sunlit_photosynthesis` when a process alone +is ambiguous. + +## Unified Model Configuration + +`ModelSpec` should become the single scenario wrapper. `MultiScaleModel(...)`, +`AllDomains(...)`, `HardDomains(...)`, and user-written `Route(...)` should be +replaced by explicit value inputs and callable model calls. + +### Applies To + +Use `AppliesTo(...)` to declare the object set where a model application runs. +This should be first-class, not inferred from a domain or mapping key. + +```julia +ModelSpec(LeafState()) |> + AppliesTo(Many(kind=:plant, scale=:Leaf)) + +ModelSpec(AllocationModel()) |> + AppliesTo(Many(kind=:plant, scale=:Plant)) + +ModelSpec(SceneEB()) |> + AppliesTo(One(scale=:Scene)) +``` + +The same model kernel can be applied several times with different selectors, +parameters, timesteps, or input bindings. The compiler should normalize each +application to a stable application id. + +### Dependency Defaults From Traits + +Model authors should still declare `inputs_`, `outputs_`, and `dep`. In the +new design, `dep(model)` becomes the model-level place for default dependency +intent, for both the current `ModelMapping` use case and the future scene/object +runtime. + +The rule is: + +- `inputs_(model)` declares the variables the model needs; +- `outputs_(model)` declares the variables the model computes; +- `dep(model)` declares default value sources or manual model calls when the + model author knows a sensible coupling pattern; +- `ModelSpec(...) |> Inputs(...)` and `ModelSpec(...) |> Calls(...)` override + or specialize those defaults for a specific simulation. + +For example, a plant allocation model can provide a plant-local default: + +```julia +dep(::PlantAllocationModel) = ( + leaf_carbon = Input(Many(scale=:Leaf, within=Self(), var=:leaf_carbon)), +) +``` + +An energy-balance model can declare that it usually calls a stomatal +conductance model manually: + +```julia +dep(::LeafEnergyBalanceModel) = ( + stomatal_conductance = Call(process=:stomatal_conductance), +) +``` + +These trait defaults are not absolute wiring. They are model-author defaults +that make common cases work without repeating configuration, while scenario +authors keep final authority through `ModelSpec`. + +Compiler order: + +1. read `inputs_`, `outputs_`, and `dep`; +2. infer simple same-object value dependencies when unambiguous; +3. apply `dep(model)` defaults for value inputs and model calls; +4. apply `ModelSpec` overrides last. + +This order is part of the public contract. It keeps modeler defaults useful +without making them final wiring. Missing or ambiguous inputs after this pass +are errors, not incidental fallback behavior. + +### Value Inputs + +Use `Inputs(...)` when a model needs values before its `run!` method executes: + +```julia +ModelSpec(LAIModel(ground_area)) |> + Inputs(:leaf_areas => Many(scale=:Leaf, within=SceneScope(), var=:leaf_area)) +``` + +Reusable plant allocation: + +```julia +ModelSpec(AllocationModel()) |> + AppliesTo(Many(scale=:Plant)) |> + Inputs(:leaf_carbon => Many(scale=:Leaf, within=Self(), var=:leaf_carbon)) +``` + +The same declaration must compile to: + +- direct `Ref`/`RefVector` wiring when producer and consumer live in the same + object graph and rate; +- temporal stream reads when producer and consumer run at different rates; +- current route materialization when target status must be assigned before a + model runs; +- source-status lookup for graph-backed object selections. + +The important user rule is: + +- `Inputs(...)` means "give this model values"; the runtime schedules or + samples producers; +- the receiving model never manually calls the producer because of an + `Inputs(...)` declaration. + +### Carrier And Copy Semantics + +The compiler chooses the carrier, but the semantics must be documented and +explainable: + +| Situation | Preferred carrier | Copy behavior | +| --- | --- | --- | +| same-rate scalar input | shared `Ref` or local alias | no copy when possible | +| same-rate `Many(...)` input | `RefVector` or equivalent typed reference collection | no copy for live values | +| cross-rate input | temporal stream sample | value materialized for the consumer timestep | +| `Integrate` or `Aggregate` input | temporal window reduction | reduced value materialized | +| route-like target status input | compiler-generated materialization | assigned before consumer run | +| environment input | cached `EnvironmentBinding` sample | backend-defined value sample | + +This table is a required part of the design because performance, units, +automatic differentiation, and error propagation depend on preserving user +value types and avoiding hidden copies. + +PlantSimEngine should not force `Float64` internally. Status values, +parameters, meteo values, and outputs must be allowed to use units, dual +numbers, uncertainty wrappers, tracked arrays, or other numeric-like types. +Compiled carriers should be parametric and type stable whenever the object set +and value type are known at initialization. + +### Multirate Inputs + +Multirate must be supported by the same `Inputs(...)` declaration, not a +separate mapping language. The public time language should remain `Dates` +periods. + +Example: + +```julia +ModelSpec(PlantAllocation()) |> + AppliesTo(Many(kind=:plant, scale=:Plant)) |> + Inputs(:leaf_assimilation => Many( + scale=:Leaf, + within=Self(), + var=:assimilation, + policy=Integrate(), + window=Day(1), + )) |> + TimeStep(Day(1)) +``` + +Policy precedence should stay explicit: + +1. input-level policy in `Inputs(...)`; +2. producer `output_policy(model)`; +3. default `HoldLast()`. + +Cross-rate links must go through temporal state even when they point to objects +that could otherwise be reference-wired. + +### Model Calls + +Use `Calls(...)` when a model must manually run selected models, typically +inside an iterative solver. This is the required public API name and must be +implemented as part of the unified scene/object redesign, not left as a later +rename. + +```julia +ModelSpec(SceneEB()) |> + Calls(:leaf_energy => Many( + kind=:plant, + scale=:Leaf, + process=:energy_balance, + )) |> + Calls(:soil => One(kind=:soil, process=:soil_water)) +``` + +Inside `run!`, the scene model receives call handles and calls +`run_call!(call; publish=false)` during trial iterations, then +`publish=true` for the accepted final solution. + +The important user rule is: + +- `Calls(...)` means "give this model callable model handles"; +- the parent model owns the call stack and can iterate, reject, or accept trial + calls; +- call outputs are published only according to the call publication contract. + +### Multiplicity + +Selection multiplicity is explicit: + +```julia +One(...) +Many(...) +OptionalOne(...) +``` + +The compiler validates that `One(...)` resolves to exactly one producer per +consumer scope. `Many(...)` returns a vector-like value or target collection. + +### Address Normalization + +All source and target declarations normalize to an internal address: + +```julia +ObjectAddress( + scope, + kind, + species, + scale, + name, + process, + var, + relation, + multiplicity, +) +``` + +Only the compiler works with this normalized address. Users should not need to +construct it manually. + +## Object Lifecycle And Spatial Contracts + +Growth, pruning, organ creation, reparenting, and moving organs must all update +the same compiled caches: + +- object selections used by `AppliesTo`, `Inputs`, and `Calls`; +- `RefVector` or equivalent many-object carriers; +- temporal stream ownership; +- writer validation; +- environment bindings. + +The public mutation API should make cache invalidation explicit and centralized: + +```julia +register_object!(scene, object; parent) +remove_object!(scene, object) +reparent_object!(scene, object, new_parent) +move_object!(scene, object, geometry_or_position) +refresh_bindings!(scene) +``` + +Spatial environment backends should depend on a small geometry contract, not on +a particular plant representation: + +```julia +position(object_or_status) +geometry(object_or_status) +bounds(object_or_status) +``` + +Packages can provide richer geometry, octrees, voxel grids, or layers, but +PlantSimEngine should only require enough information to bind an object to an +environment provider. + +## Duplicate Writers And Updates + +Most variables should have one canonical writer per object and timestep. When a +variable is intentionally updated by several models, the scenario should say so +where the model applications are assembled: + +```julia +ModelSpec(PruningModel()) |> + AppliesTo(Many(scale=:Leaf)) |> + Updates(:leaf_biomass; after=:carbon_allocation) +``` + +`Updates(...)` should be rare and explicit. It is a scenario-level ordering +rule, because a model author cannot predict every model that will later update +the same variable. + +## Environment And Microclimate + +Meteorology should remain automatic unless a model or scenario needs special +behavior. Models declare environment variables: + +```julia +meteo_inputs_(::LeafEnergyModel) = ( + T=0.0, + Rh=0.0, + Wind=0.0, + Ri_PAR_f=0.0, + CO2=0.0, +) +``` + +The runtime resolves those variables through the scene environment service. + +Default resolution: + +1. A global/table meteo backend gives every object the current meteo row. +2. A voxel, octree, layered, or grid backend samples the cell bound to the + object. +3. If the object has no position, use the parent position. +4. If no spatial binding can be made, fall back to global meteo or error when + the environment variable is required. + +Users can override the binding contract: + +```julia +EnvironmentResolver( + bind=(scene, object) -> containing_cell(scene.microclimate, position(object)), +) +``` + +PlantSimEngine should define the protocol and caching hooks, not the voxel or +octree implementation. Specialized packages should provide concrete spatial +backends. + +The environment backend protocol should be small and backend-oriented: + +```julia +bind_environment(scene, backend, object) +sample_environment(backend, binding, time, variables) +scatter_environment!(backend, binding, values) +refresh_environment!(backend, scene) +``` + +`meteo_inputs_(model)` declares what a model reads from the active environment +provider. `meteo_outputs_(model)` declares what a model can write back to a +mutable microclimate provider. Simple global meteorology remains the default +provider. + +### Cached Environment Bindings + +Spatial lookup must not happen for every model call. At initialization and when +objects are created, the runtime builds an environment binding cache: + +```julia +EnvironmentBinding( + object_id, + provider=:microclimate_grid, + cell_id, + variables=(:T, :Rh, :Wind, :Ri_PAR_f), +) +``` + +Runtime sampling is: + +```text +object -> cached binding -> environment cell -> current values +``` + +Invalidation events: + +- object created; +- object removed; +- object moved; +- geometry changed; +- environment grid rebuilt or refined; +- model environment requirements changed. + +Geometry APIs should provide ergonomic invalidation: + +```julia +mark_environment_binding_dirty!(scene, object) +update_geometry!(object, geometry; invalidate_environment=true) +``` + +Before each timestep, dirty bindings are refreshed in batch. + +## Compilation Strategy + +The compiler should build one global dependency graph over object addresses. +The graph includes: + +- value dependencies from `Inputs(...)`; +- callable dependencies from `Calls(...)`; +- model update edges from `Updates(...)`; +- temporal policy edges; +- environment reads and writes; +- object-scope selection caches. + +The runtime representation is an implementation detail: + +- same-rate local links can stay as aliases; +- cross-rate links use temporal state; +- many-object links use `RefVector` or node-value streams; +- call links use `ModelCall` or an equivalent callable runtime handle; +- environment links use cached `EnvironmentBinding`s. + +The public explanation API must describe the normalized graph, not the internal +carrier choice. + +## Agent-Facing Requirements + +The final design must be understandable by agents through structured +explanation helpers: + +```julia +explain_objects(scene) +explain_instances(scene) +explain_scopes(scene) +explain_bindings(sim) +explain_calls(sim) +explain_environment_bindings(sim) +explain_schedule(sim) +explain_writers(sim) +``` + +These helpers should return stable structured data, not only pretty text. A +binding row should include at least: + +- consumer application id; +- consumer object id; +- consumer variable; +- source selector; +- resolved producer application id or environment provider id; +- resolved producer object ids; +- process/name filters; +- temporal policy and window; +- carrier kind; +- copy/reference semantics; +- reason the binding was chosen; +- whether it came from inference, `dep(model)`, or `ModelSpec`. + +Errors should report concrete object labels, scope selectors, process names, +variables, and suggested fixes. + +## Compatibility Position + +This is a breaking target design. It should preserve model kernels and the +`run!(model, models, status, meteo, constants, extra)` contract when possible, +but it may replace the scenario configuration surface: + +- `MultiScaleModel(...)` becomes `Inputs(...)`; +- `Route(...)` becomes a compiler-generated carrier for `Inputs(...)`; +- `AllDomains(...)` becomes a selector used inside `Inputs(...)`; +- `HardDomains(...)` becomes `Calls(...)`; +- `Domain(...)` becomes an object scope/template/instance concept; +- `InputBindings(...)` becomes explicit policy and source information on + `Inputs(...)`; +- `MeteoBindings(...)` becomes automatic environment binding plus optional + `Environment(...)` overrides; +- `OutputRouting(...)` remains model-application output configuration or is + folded into a clearer output policy modifier; +- `PreviousTimeStep(...)` remains a temporal policy/cycle-breaking marker in + the unified graph. diff --git a/docs/src/dev/unified_scene_object_implementation_plan.md b/docs/src/dev/unified_scene_object_implementation_plan.md new file mode 100644 index 000000000..1ba7e8df7 --- /dev/null +++ b/docs/src/dev/unified_scene_object_implementation_plan.md @@ -0,0 +1,744 @@ +# Unified Scene/Object Implementation Plan + +This plan is the persistent handoff for replacing the current domain/route and +multiscale-mapping split with one scene/object address system. + +The implementation can be incremental internally, but the target API is +breaking. Do not try to preserve `MultiScaleModel(...)`, `Route(...)`, +`AllDomains(...)`, or `HardDomains(...)` as primary user-facing concepts in the +final design. + +The target public surface should be centered on a small set of concepts: + +```julia +Scene +Object +ModelSpec + +AppliesTo(...) +Inputs(...) +Calls(...) +Updates(...) +TimeStep(...) +Environment(...) +``` + +This is the API memory target for users, modelers, and agents. Additional +types should be selectors, model traits, or internal compiled carriers. + +## Current State To Preserve Until Replacement + +The current experimental branch already implements useful behavior: + +- `Domain`, `SimulationMapping`, and `DomainSimulation`; +- single-status and MTG-backed domains; +- graph-domain selectors for several plant instances/species; +- `Route(...)` materialization into scene status; +- `AllDomains(...)` stream/value dependencies; +- `HardDomains(...)` targets and `run_target!`; +- multi-rate domain scheduling with `Dates`; +- environment backend protocol and constant meteo backend; +- dynamic MTG add/remove/reparent support; +- `Updates(...)` for ordered duplicate writers; +- structured domain explanation helpers. + +Those features are the behavioral test bed for the new design. The new API +should reproduce the same capabilities through unified object selections. + +## Implementation Progress + +- Started Phase 0 by adding the public API vocabulary as real typed metadata: + `AppliesTo(...)`, `Inputs(...)`, `Calls(...)`, `TimeStep(...)`, and + `Environment(...)` can now be applied to `ModelSpec`. +- Added `ModelSpec(model; name=...)` application names and getters: + `application_name`, `applies_to`, `value_inputs`, `model_calls`, and + `environment_config`. +- Added initial selector and address types: + `SceneScope`, `Self`, `SelfPlant`, `Ancestor`, `Scope`, `Kind`, `Species`, + `Scale`, `Relation`, `One`, `OptionalOne`, `Many`, and `ObjectAddress`. +- Added initial `Scene`/`Object` registry types and lifecycle hooks: + `register_object!`, `remove_object!`, `reparent_object!`, `move_object!`, + and `refresh_bindings!`. +- Added registry-backed selector resolution with `resolve_object_ids` and + `resolve_objects` for `SceneScope()`, `Self()`, `SelfPlant()`, + `Ancestor(...)`, `Scope(...)`, positional selectors such as + `Kind(:plant)`/`Scale(:Leaf)`, and `One`/`OptionalOne`/`Many` cardinality + checks. +- Added `explain_scopes(scene)` for agent-readable scope diagnostics. It + reports the global scene scope, each object subtree, each named + `Scope(...)`, and label groups by scale, kind, and species with concrete + resolved object ids. +- Started the object-address compiler with `compile_scene(scene, specs)` and + compiled scene application/binding carriers. The compiler now resolves + `AppliesTo(...)` target object ids, object-relative `Inputs(...)` source + object ids, and object-relative `Calls(...)` callee object/application ids + before runtime. +- Added `explain_scene_applications`, `explain_bindings`, and `explain_calls` + for the compiled scene view. These explanations expose application ids, + processes, target ids, input source ids, call callee ids, temporal policy, + window, and carrier hints. +- Added status-backed compiled input carriers. When source objects already + hold `Status` values, `Inputs(...)` bindings now precompile a scalar shared + `Ref`, a homogeneous `RefVector`, or an `ObjectRefVector` fallback for + heterogeneous reference-preserving vectors. `input_carrier`, `input_value`, + and `has_reference_carrier` expose these carriers, and `explain_bindings` + reports carrier kind, copy/reference semantics, carrier type, and reference + availability. +- Added conservative same-object input inference in the scene compiler. When a + model declares an `inputs_` variable that is not covered by explicit/default + `Inputs(...)`, and exactly one other application on the same object outputs + the same variable, `compile_scene` creates an inferred reference binding. + `explain_bindings` now reports binding `origin` values such as + `:declared` and `:inferred_same_object`. +- Compiled input bindings now carry producer metadata. When an `Inputs(...)` + selector uses `process=` or `application=`, `compile_scene` validates that a + matching source application exists for the selected source objects. + `explain_bindings` reports `source_application_ids`, `process`, and + `application` for agent-readable dependency diagnostics. +- Dependency selectors in `Inputs(...)` and `Calls(...)` now infer a default + scope from the consumer object when no explicit `within=...` is provided: + scene objects default to `SceneScope()`, while non-scene objects default to + `Self()`. Shared scene/soil dependencies from organs should therefore use + `within=SceneScope()` explicitly. +- `compile_scene` now validates required status inputs from `inputs_(model)`. + Each required input must either have a compiled binding or already exist on + the target object `Status`; otherwise compilation errors with the concrete + application id, object id, and input variable. +- `compile_scene` now rejects `Inputs(...)` declarations whose left-hand + variable is not declared by the target model's `inputs_`. This catches + misspelled or stale scenario bindings before they create silent unused + metadata. +- `compile_scene` now validates source availability for status-backed + non-temporal `Inputs(...)` bindings. When selected source objects already + have `Status` values, the requested source variable must resolve to + references instead of silently compiling to an unused/no-op binding. +- Carrier compilation preserves source `Status` references and arbitrary value + types; tests cover both scalar refs and many-object vectors with a custom + non-`Float64` value type. +- Added call ambiguity validation in the compiled scene view: a call can select + by process when unique, and must use `application=:name` when several model + applications with the same process match the same object. +- Added a scene binding cache with `refresh_bindings!`, `bindings_dirty`, + `compiled_bindings`, and `scene_revision`. Object creation, removal, + and reparenting now invalidate the compiled binding cache and bump a scene + revision before the next refresh. +- Added an environment binding cache with `refresh_environment_bindings!`, + `compile_environment_bindings`, `CompiledEnvironmentBinding`, + `CompiledEnvironmentBindings`, `environment_bindings_dirty`, + `compiled_environment_bindings`, `environment_revision`, and + `explain_environment_bindings`. The compiler resolves each + application/object environment provider, backend, required + `meteo_inputs_`, produced `meteo_outputs_`, support descriptor, and backend + cell before runtime. +- Added the minimal scene geometry contract: `geometry(object_or_status)`, + `position(object_or_status)`, and `bounds(object_or_status)`. Environment + binding refreshes now call `update_index!(backend, entities)` once per + distinct backend before `bind_environment`, giving spatial backends a current + scene-wide object/entity list for precomputed microclimate lookup. +- Object creation, removal, and reparenting invalidate both structural and + environment bindings. Object movement invalidates only environment bindings, + so moving a leaf or changing its geometry can refresh microclimate lookup + without rebuilding object/model binding carriers. +- Added public geometry invalidation helpers: + `update_geometry!(scene, object, geometry; invalidate_environment=true)` + and object-scoped `mark_environment_binding_dirty!(scene, object)`. + These currently route to the scene environment binding cache invalidation; + finer-grained per-object dirty tracking can be added behind the same API. +- Started scene/object execution with `run!(scene; steps=...)`. + The runtime refreshes compiled object bindings and environment bindings, + materializes precompiled `Inputs(...)` carriers into consumer `Status` + fields, samples the bound environment backend, and calls generic model + kernels through the existing `run!` contract. +- Scene/object execution now publishes model outputs to scene-local temporal + streams. Compiled `Inputs(...)` bindings marked as `:temporal_stream` can + materialize `HoldLast`, `Integrate`, and `Aggregate` values before the + consumer runs, using selector source ids, source variables, windows, and the + scene base timestep. +- Scene/object execution now scatters mutable environment outputs declared by + `meteo_outputs_(model)` back to the bound backend after each model call. + This reuses `scatter_environment_outputs!`, so environment writers keep the + existing generic model contract: compute a same-named status value, and let + the runtime push it to the active microclimate backend. +- Added root application scheduling from `TimeStep(...)` using `Dates.Period` + values and the scene environment base step. `explain_schedule` on a + `CompiledScene` now reports each application clock, phase, timestep in base + steps, timestep duration in seconds, and whether the application is scheduled + as a root application or is manual-call-only. +- `compile_scene` now computes a stable topological application order from + resolved `Inputs(...)` producer edges and `Updates(...)` writer-order edges. + Inputs produced by manual-call-only applications are redirected to the parent + application that owns the `Calls(...)` call stack. `run!(scene)` uses this + precompiled order instead of user declaration order, cycles fail at compile + time, and `explain_schedule` reports `execution_index`. +- `CompiledScene` now pre-indexes input and call bindings by + `(application_id, object_id)`. Per-object input materialization and + `dependency_target(s)` lookup use these indexes instead of scanning every + binding in the scene at each model call. +- `CompiledScene` now also pre-indexes applications by application id. + Hard-call target resolution and stable ordered-application materialization + use this index instead of scanning or rebuilding lookup dictionaries. +- `CompiledEnvironmentBindings` now pre-indexes environment bindings by + `(application_id, object_id)`. Environment sampling and mutable environment + output scattering use direct lookup instead of scanning all environment + bindings for every model invocation. +- Added `SceneRunContext` and `SceneCallTarget`. Models can retrieve manual + `Calls(...)` targets with `dependency_target(s)(extra, :name)` and execute + them with `run_call!`, preserving explicit call-stack control in the + scene/object runtime. Manual calls execute immediately under the parent call + stack; applications selected by `Calls(...)` are skipped by the root + `run!(scene)` loop and only execute through `run_call!`. +- Added scene/object duplicate-writer validation. During `compile_scene`, each + `(object, output variable)` now has one canonical writer unless later + writers declare `Updates(:var; after=...)`. The `after` token can match a + previous application id/name or process, so scenario authors can express + cases such as pruning after carbon allocation without changing either model + implementation. +- Added `explain_writers(compiled)`. It reports each object/variable writer + group, duplicate-writer status, writer application ids/processes, and the + `Updates(...)` declarations used to validate ordered updates. +- Extended `explain_model_specs` rows with application name, target selector, + value inputs, manual calls, and environment metadata. +- Started Phase 3 by bridging simple `Inputs(...)` declarations into the + existing MTG multiscale mapping carrier when the selector is representable as + a pure scale/variable mapping, for example + `Inputs(:x => Many(scale=:Leaf, var=:y))`. +- Added model-level `Input(...)` defaults from `dep(model)` into + `ModelSpec` value inputs. Scenario-level `ModelSpec(...) |> Inputs(...)` + overrides those defaults and also replaces the corresponding internal legacy + mapping carrier for the same input. +- Extended the Phase 3 bridge for domain simulations by generating internal + `Route(...)` carriers from supported consumer-side `Inputs(...)` + declarations, for example + `Inputs(:leaf_areas => Many(kind=:plant, scale=:Leaf, process=:leaf_state, var=:leaf_area))`. +- Started Phase 4 by bridging `Calls(...)` declarations into the current + hard-domain dependency resolver when selectors can be represented by + `kind`, `domain`, `scale`, and `process`. Added `run_call!` as the unified + spelling over the current `ModelTarget` execution path. +- Added model-level `Call(...)` defaults from `dep(model)` into + `ModelSpec` manual-call metadata. Scenario-level + `ModelSpec(...) |> Calls(...)` overrides those defaults, and + `dep(::ModelSpec)` excludes raw `Call(...)` trait entries so default calls + are normalized through the same bridge as explicit calls. +- Migrated the MAESPA example's scene energy-balance hard calls from + model-level `HardDomains(...)` to scenario-level + `ModelSpec(scene_model) |> Calls(...)`. +- Migrated the MAESPA example's scene LAI leaf-area route from user-written + `Route(...)` to consumer-side `ModelSpec(LAIModel(...)) |> Inputs(...)`. +- Started Phase 5 with `ObjectTemplate` and `ObjectInstance`. A template stores + reusable scene/object `ModelSpec`s plus default object labels, and an + instance mounts those specs inside one named object subtree. +- `Scene(...)` accepts `ObjectInstance` values directly or through its + `instances` keyword. An instance root can be an owned `Object` or the id of + an object supplied separately to the scene. +- Mounted template applications receive stable instance-prefixed application + names and an implicit `Scope(instance_name)` on unqualified + `AppliesTo(...)` selectors. Their `Inputs(...)`, `Calls(...)`, scheduling, + writer validation, and execution use the normal compiled scene/object path. +- Instance overrides can replace one template application by application name + or process. Overrides must be unambiguous and preserve process identity. + Instances without overrides retain the exact shared model object from the + template. +- Template labels fill missing `kind` and `species` metadata throughout the + mounted subtree, while the root receives the instance name used by + `Scope(...)`. Tests cover four instances, plant-local aggregation, shared + model storage, and one process-level model override. +- Added explicit exceptional-organ overrides with + `Override(object=..., application=... or process=..., model=...)` through + `ObjectInstance(...; object_overrides=...)`. The override must resolve to one + template application, belong to the instance subtree, and preserve process, + input, output, and environment-variable declarations. +- Object overrides remain one logical model application: the compiler stores + the selected replacement model by target object id. Dependency bindings, + writer ownership, application names, and manual calls therefore remain + unchanged, and no selector resolution occurs in the runtime loop. +- Parameter/model ownership is explicit. Templates retain user-supplied model + and `parameters` objects by reference; unchanged instances share them. + Instance and object overrides retain their user-supplied replacement model + by reference. PlantSimEngine does not copy models or mutate model fields to + merge parameter overrides. +- Same-concrete-type object overrides use a concretely typed object-to-model + table. Structured application explanations report shared/per-object storage, + concrete versus heterogeneous dispatch, overridden object ids, and model + types. +- `Scene` retains mounted instance metadata and `explain_instances(scene)` + reports each instance root, current subtree object ids, mounted application + ids, instance/object overrides, template labels, and parameter ownership. + `explain_objects(scene)` also reports instance membership. +- New objects registered below an instance automatically inherit missing + template `kind` and `species` labels. Instance explanations derive membership + from the current topology, so growth, pruning, and reparenting do not leave a + separate stale membership list. + +This progress is still a bridge over the existing compiler, not the final +object-address compiler. Supported `Inputs(...)` and `Calls(...)` selectors are +translated to current carriers where possible. Unsupported object-relative +selectors remain structured metadata until the object-address compiler lands. + +## Phase 0: Public Contract Freeze + +Goal: decide the small public vocabulary before implementing internals. + +Define: + +- `ModelSpec(model; name=nothing)` as the model-application wrapper. +- `AppliesTo(selector)` as the target object-set declaration. +- `Inputs(...)` for value dependencies. +- `Calls(...)` for manual call-stack dependencies. +- `Updates(...)` for rare ordered duplicate writers. +- `TimeStep(period::Dates.Period)` and related multirate policies. +- `Environment(...)` for optional environment resolver/backend overrides. + +Rules: + +- a model kernel remains generic and declares `inputs_`, `outputs_`, optional + `dep`, optional `meteo_inputs_`/`meteo_outputs_`, and `run!`; +- a model application decides where the kernel runs, at what rate, and how its + inputs, calls, updates, outputs, and environment are bound; +- application ids are stable and can be generated from explicit `name`, + process, object selector, and occurrence index; +- if several applications provide the same process on the same object set, + selectors must disambiguate by application name or another explicit filter. + +Acceptance tests: + +- a model can be applied twice to the same leaf objects with different names; +- a dependency selector can choose by process when unique and by name when not; +- structured explanations expose model kernel type, process, application name, + and target object ids. + +## Phase 1: Scene Object Registry + +Goal: introduce the internal object model without changing public behavior yet. + +Implement: + +- `ObjectId` as the stable identity key for every runtime object. +- `SceneObject` metadata with labels: + `scale`, `kind`, `species`, optional `name`, parent id, child ids, and + optional geometry/position handle. +- `SceneRegistry` storing objects, parent/child relations, and indexes by + label. +- adapters from the current MTG/domain state into the registry: + each selected domain root and each MTG node gets an object id; + single-status domains get one object with `scale=:Default`. +- object lifecycle hooks for add/remove/reparent that mirror the existing MTG + runtime reindexing. + +Acceptance tests: + +- the MAESPA example registers five leaf objects, two plant objects, one soil + object, and one scene object; +- `status(sim, :plant_A, :Leaf)` and `status(sim, :Leaf)` can be expressed as + registry queries; +- add/remove/reparent updates object registry relations and status views. + +## Phase 2: Selector And Scope Language + +Goal: make "which objects?" explicit and reusable. + +Implement selector types: + +```julia +SceneScope() +Self() +SelfPlant() +Ancestor(scale=:Plant) +Scope(name) +Kind(kind) +Species(species) +Scale(scale) +Relation(...) +``` + +Implement multiplicity wrappers: + +```julia +One(selector...) +OptionalOne(selector...) +Many(selector...) +``` + +Selectors must normalize to `ObjectAddress` objects with enough context to be +resolved relative to a consuming object. + +Implement `AppliesTo(...)` using the same selector system. The target object +set of a model application must never be hidden inside a domain key, mapping +key, or implicit scale table. + +Definitions: + +- `Self()` means the current model application object or scope. It is the + current plant only when the model is applied at `scale=:Plant`. +- `SelfPlant()` means the nearest containing plant scope. +- `Ancestor(scale=:Plant)` is the generic selector form for `SelfPlant()`. +- `SceneScope()` means the whole scene. +- `Scope(name)` means a named scope or object collection. + +Rules: + +- unqualified selectors inside a reusable plant mapping default to + `within=Self()`; +- scene-level selectors default to `within=SceneScope()`; +- `One(...)` errors unless exactly one object resolves per consumer; +- `Many(...)` preserves stable object-id order; +- object-id order replaces incidental traversal order as the semantic default. +- selectors are resolved during compilation or binding refresh, not inside the + inner model loop. + +Acceptance tests: + +- plant allocation on four oil palms reads only leaves under each plant; +- scene LAI reads leaves across all plant objects; +- a species-specific scene model can read only `species=:oil_palm` leaves; +- a model application target set declared with `AppliesTo(...)` produces stable + application/object pairs; +- selector errors report available labels and near matches. + +## Phase 3: Unified Value Inputs + +Goal: replace `MultiScaleModel(...)` and user-written `Route(...)` with +`Inputs(...)`, while using the existing `dep` trait as the model-level source +of default dependency intent. + +Target API: + +```julia +ModelSpec(AllocationModel()) |> + AppliesTo(Many(kind=:plant, scale=:Plant)) |> + Inputs(:leaf_carbon => Many(scale=:Leaf, within=Self(), var=:leaf_carbon)) + +ModelSpec(LAIModel(area)) |> + AppliesTo(One(scale=:Scene)) |> + Inputs(:leaf_areas => Many(kind=:plant, scale=:Leaf, within=SceneScope(), var=:leaf_area)) +``` + +Implement: + +- `Inputs(...)` as `ModelSpec` configuration. +- `Input(...)` or an equivalent internal wrapper that lets `dep(model)` + provide default value-input bindings. +- normalized input bindings from target variable to `ObjectAddress`. +- compiler pass that decides carrier: + direct reference, `RefVector`, temporal stream, or route-like + materialization. +- status-default insertion for materialized target variables using the + consumer model's `inputs_` default. +- temporal policies on value inputs: + `HoldLast`, `Interpolate`, `Integrate`, `Aggregate`. +- `Dates.Period` windows on value inputs, for example `window=Day(1)`. +- copy/reference semantics reporting for every compiled input binding. + +Rules: + +- model authors still declare `inputs_`; scenario authors decide where those + inputs come from; +- `dep(model)` may provide defaults for common value-input bindings, for both + current `ModelMapping`-style composition and future scene/object composition; +- scenario-level `ModelSpec(...) |> Inputs(...)` always wins over `dep(model)` + defaults; +- same-rate local links should keep reference semantics where possible; +- cross-rate links always go through temporal state; +- duplicate source candidates are errors unless the selector disambiguates; +- route structs may remain internally but should not be required in examples. +- same-rate scalar and many-object links should avoid copies when they can use + aliases, shared refs, `RefVector`, or an equivalent typed carrier; +- PlantSimEngine must preserve arbitrary value types, including units, + automatic differentiation numbers, uncertainty wrappers, and other + numeric-like values. + +Carrier expectations: + +| Binding kind | Runtime carrier | +| --- | --- | +| same-rate scalar | shared `Ref` or local alias | +| same-rate many-object | `RefVector` or equivalent typed reference collection | +| cross-rate | temporal stream sample | +| integrate/aggregate | temporal window reduction | +| materialized route | generated pre-run status assignment | +| environment | cached environment binding sample | + +Acceptance tests: + +- the MAESPA scene LAI route becomes an `Inputs(...)` declaration and produces + the same `lai` and `leaf_area`; +- current plant allocation `MultiScaleModel([:leaf_carbon => [:Leaf => :leaf_carbon]])` + becomes `Inputs(...)` and remains plant-local; +- a same-scale rename currently expressed with `SameScale()` works through + `Inputs(...)`; +- multi-rate value inputs integrate graph-domain node streams by object id. +- same-rate many-object bindings do not allocate per timestep in a benchmarked + hot loop beyond unavoidable model work. +- unitful or dual-number status values survive `Inputs(...)` without forced + conversion to `Float64`. + +## Phase 4: Unified Model Calls + +Goal: replace `HardDomains(...)` with `Calls(...)`. `Calls(...)` is the +required public API for manually controlled model execution and must be +implemented in this phase, not deferred to a future rename. The same mechanism +must also be usable from `dep(model)` so current hard-dependency traits become +default call declarations. + +Target API: + +```julia +ModelSpec(SceneEB()) |> + AppliesTo(One(scale=:Scene)) |> + Calls(:leaf_energy => Many(kind=:plant, scale=:Leaf, process=:energy_balance)) |> + Calls(:soil => One(kind=:soil, process=:soil_water)) +``` + +Implement: + +- `Calls(...)` as `ModelSpec` configuration. +- `Call(...)` or an equivalent internal wrapper that lets `dep(model)` provide + default manual-call dependencies. +- call resolution from `ObjectAddress` to concrete `ModelCall` handles, or an + equivalent callable runtime object if the final internal type name differs. +- same-status hard dependency calls using the same public API. +- publication semantics: + trial `run_call!(call)` mutates status only; + final `run_call!(call; publish=true)` appends outputs and temporal + streams. +- structured call explanations with parent application id, selected callee + application ids, selected object ids, selector, and publication behavior. + +Rules: + +- calls are manual call-stack dependencies and are not independently + scheduled under the parent; +- `dep(model)` call defaults are model-author defaults, not final wiring; +- scenario-level `ModelSpec(...) |> Calls(...)` overrides `dep(model)` defaults; +- hard target outputs still participate in dependency graph compilation through + the owning parent when needed; +- call selection must be visible through explanation helpers. + +Acceptance tests: + +- MAESPA scene energy balance uses `Calls(...)` and still controls iterative + leaf energy calls; +- missing call selectors report `kind`, `scale`, `process`, and available + matches; +- final accepted calls publish exactly once per timestep. +- an iterative scene model can run selected leaf and soil calls several times + with `publish=false` and publish only the accepted state. + +## Phase 5: Object Templates, Instances, And Overrides + +Goal: support several plants of the same species with shared default models and +selective per-instance differences. + +Target API: + +```julia +oil_palm = ObjectTemplate( + kind=:plant, + species=:oil_palm, + mapping=oil_palm_mapping, +) + +scene = Scene( + ObjectInstance(:palm_1, oil_palm; root=node1), + ObjectInstance(:palm_2, oil_palm; root=node2, overrides=( + stomatal_conductance = Tuzet(; g1=3.2), + )), +) +``` + +Implement: + +- template-level model specs and parameters; +- instance-level model/parameter overrides by process; +- object-level overrides for exceptional organs; +- conflict validation when two overrides target the same process/object. +- shared parameter/model storage when template instances do not override + anything, with explicit copy/ownership behavior when they do. + +Rules: + +- templates do not prescribe topology; they attach mappings to whatever object + tree the instance provides; +- default `Self()` selectors resolve inside the current instance; +- scene-wide models must opt into wider scope. + +Acceptance tests: + +- four oil palm instances share model objects/parameters when not overridden; +- one palm instance can override one process parameter; +- allocation remains per plant while scene LAI sees all leaves. + +Current remaining work: + +- migrate the repeated plant construction in the MAESPA example to templates + once its complete object topology is represented by the unified runtime. + +## Phase 5B: Object Lifecycle And Cache Invalidation + +Goal: make growth, pruning, and moving organs update every compiled binding +through one mutation path. + +Implement public lifecycle hooks: + +```julia +register_object!(scene, object; parent) +remove_object!(scene, object) +reparent_object!(scene, object, new_parent) +move_object!(scene, object, geometry_or_position) +refresh_bindings!(scene) +``` + +Implement invalidation for: + +- object selector caches; +- model application target sets; +- `RefVector` or equivalent many-object carriers; +- temporal stream ownership; +- writer validation; +- environment bindings. + +Rules: + +- topology and geometry changes do not silently leave stale carriers; +- object creation should bind the new object to model applications selected by + `AppliesTo(...)` before the next timestep; +- moving an object should refresh environment bindings without rebuilding + unrelated model bindings unless the move changes object relations or labels. + +Acceptance tests: + +- creating a new leaf adds it to plant-local allocation and scene LAI before + the next timestep; +- pruning/removing a leaf removes it from many-object carriers and temporal + stream ownership; +- changing a leaf insertion angle can refresh only the affected environment + binding when topology is unchanged. + +## Phase 6: Environment Binding Cache + +Goal: make meteo/microclimate automatic and fast. + +Implement: + +- `EnvironmentBinding` cache: + object id, backend/provider id, cell/layer id, required variables. +- default environment resolver: + global meteo for non-spatial backends; + object position for spatial backends; + parent position fallback; + global fallback or validation error. +- dirty flags and batched refresh: + `mark_environment_binding_dirty!`; + `update_geometry!(...; invalidate_environment=true)`; + automatic dirty marking on object creation, removal, reparenting, and + environment grid rebuild. +- explanation helper: + `explain_environment_bindings(sim)`. +- minimal geometry accessors or traits: + `position`, `geometry`, and `bounds`. +- backend protocol: + `bind_environment`, `sample_environment`, `scatter_environment!`, and + `refresh_environment!`. +- `Environment(...)` overrides for scenario-specific resolver/backend choices. + +Runtime rule: + +```text +object -> cached binding -> backend cell/layer -> current meteo values +``` + +Spatial lookup must happen only during binding refresh, not inside every model +call. + +Acceptance tests: + +- global meteo gives the same values to all objects; +- mock grid backend binds leaves to cells once at initialization; +- moving one leaf marks only that leaf binding dirty and refreshes it before + the next timestep; +- model `meteo_inputs_` changes update required variables without recomputing + spatial links unless necessary. +- `meteo_outputs_` can write mutable microclimate variables back to the active + backend through `scatter_environment!`. + +## Phase 7: Compiler, Scheduler, And Explanation Cleanup + +Goal: make the unified graph the source of truth. + +Implement: + +- one compiler that builds a global dependency graph over object addresses; +- route/materialization and multiscale reference wiring as internal carriers; +- domain DAG scheduling replaced by object/scope dependency scheduling; +- writer validation through the same graph, including `Updates(...)`; +- model application scheduling from `AppliesTo(...)` target sets; +- multirate scheduling based on `Dates.Period` values in `TimeStep(...)` and + input windows; +- typed compiled bindings that avoid selector resolution in timestep hot loops; +- arbitrary value type preservation through status, input carriers, temporal + storage, and environment samples; +- structured explanation: + `explain_objects`, `explain_scopes`, `explain_bindings`, + `explain_calls`, `explain_environment_bindings`, `explain_schedule`, + `explain_writers`. + +Acceptance tests: + +- old `MultiScaleModel` examples rewritten with `Inputs(...)` produce matching + outputs; +- old `Route(...)` examples rewritten with `Inputs(...)` produce matching + outputs; +- MAESPA hard-call example rewritten with `Calls(...)` produces matching + outputs; +- explanation helpers include enough concrete object ids, scales, processes, + and variables for an AI agent to repair bad mappings. +- `explain_bindings(sim)` reports whether each dependency came from inference, + `dep(model)`, or `ModelSpec`, and reports carrier/copy semantics. +- no selector resolution occurs inside the per-object, per-model timestep loop + for static scenes. +- multirate simulations use the same object-address graph as same-rate + simulations. + +## Phase 8: Breaking API Removal And Migration Docs + +Goal: remove the old configuration surface once parity is proven. + +Remove or demote: + +- `MultiScaleModel(...)` as public scenario configuration; +- public `Route(...)` authoring for normal value inputs; +- `AllDomains(...)` and `HardDomains(...)` as primary user API; +- `Domain(...)` as a user-facing container when object templates/instances are + available. + +Write migration notes: + +- `MultiScaleModel([:x => [:Leaf => :y]])` -> `Inputs(:x => Many(scale=:Leaf, var=:y))`; +- `Route(from=AllDomains(...), to=...)` -> consumer `Inputs(...)`; +- `HardDomains(...)` -> `Calls(...)`; +- repeated species domains -> `ObjectTemplate` plus `ObjectInstance`; +- explicit meteo routes -> environment resolver/binding backend. +- `InputBindings(...)` -> source and temporal policy information inside + `Inputs(...)`; +- `MeteoBindings(...)` and `MeteoWindow(...)` -> `Environment(...)` and + environment sampling/window policy; +- `OutputRouting(...)` -> model-application output policy; +- `PreviousTimeStep(...)` -> temporal policy/cycle-breaking marker in the + unified graph; +- `ScopeModel(...)` -> `AppliesTo(...)` plus selector scope. + +Regression tests must cover all migrated examples before removal. + +## Open Decisions + +- Final names for `SceneScope`, `Self`, `SelfPlant`, `Kind`, `Species`, and + `Scale`. +- Whether `Inputs(...)` should be a pipeable `ModelSpec` modifier only, or also + accepted as a keyword in `ModelSpec`. +- Whether `TimeStep(...)` is the final name, or whether the existing + `TimeStepModel(...)` should be kept as an alias during the transition. +- Whether `Environment(...)` owns only backend/resolver choices or also + per-model meteo window policies. +- Whether object templates should own topology construction helpers or only + consume prebuilt MTGs/object trees. +- How much of the old API remains as compatibility wrappers after the breaking + release. diff --git a/docs/src/domain_simulation.md b/docs/src/domain_simulation.md new file mode 100644 index 000000000..914c2ef42 --- /dev/null +++ b/docs/src/domain_simulation.md @@ -0,0 +1,366 @@ +# Domain simulations + +Domain simulations let you reuse complete plant, soil, scene, or environment +model mappings without renaming their internal scales and processes. + +This is useful when a scene contains several plant species. Each species can +keep its own `ModelMapping`, parameters, hard dependencies, multiscale mappings, +and rates. A `SimulationMapping` then assembles these domains and makes +cross-domain coupling explicit. + +The example on this page is deliberately simple. The numbers are arbitrary and +chosen only to show domain coupling and multi-rate execution. + +## A two-plant scene with shared soil + +First we define a few small models. The plant domains compute absorbed radiation +and transpiration hourly. The soil domain computes water content and evaporation +hourly. The scene domain runs daily and consumes all plant transpiration plus +soil evaporation through explicit `AllDomains(...)` value dependencies. These +dependencies read already-published producer streams; they do not manually run +the producer models. + +```@example domains +using PlantSimEngine +using PlantMeteo +using Dates + +PlantSimEngine.@process "doc_domain_absorbed_radiation" verbose=false +PlantSimEngine.@process "doc_domain_plant_transpiration" verbose=false +PlantSimEngine.@process "doc_domain_soil_water" verbose=false +PlantSimEngine.@process "doc_domain_soil_evaporation" verbose=false +PlantSimEngine.@process "doc_domain_scene_evapotranspiration" verbose=false + +struct DocDomainAbsorbedRadiationModel <: AbstractDoc_Domain_Absorbed_RadiationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DocDomainAbsorbedRadiationModel) = NamedTuple() +PlantSimEngine.outputs_(::DocDomainAbsorbedRadiationModel) = (absorbed_radiation=0.0,) +PlantSimEngine.meteo_inputs_(::DocDomainAbsorbedRadiationModel) = (Ri_PAR_f=0.0,) + +function PlantSimEngine.run!(model::DocDomainAbsorbedRadiationModel, models, status, meteo, constants=nothing, extra=nothing) + status.absorbed_radiation = model.coefficient * meteo.Ri_PAR_f + return nothing +end + +struct DocDomainPlantTranspirationModel <: AbstractDoc_Domain_Plant_TranspirationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DocDomainPlantTranspirationModel) = (absorbed_radiation=0.0,) +PlantSimEngine.outputs_(::DocDomainPlantTranspirationModel) = (transpiration=0.0,) + +function PlantSimEngine.run!(model::DocDomainPlantTranspirationModel, models, status, meteo, constants=nothing, extra=nothing) + status.transpiration = model.coefficient * status.absorbed_radiation + return nothing +end + +struct DocDomainSoilWaterModel <: AbstractDoc_Domain_Soil_WaterModel + baseline::Float64 +end + +PlantSimEngine.inputs_(::DocDomainSoilWaterModel) = NamedTuple() +PlantSimEngine.outputs_(::DocDomainSoilWaterModel) = (soil_water_content=0.0,) +PlantSimEngine.meteo_inputs_(::DocDomainSoilWaterModel) = (T=0.0,) + +function PlantSimEngine.run!(model::DocDomainSoilWaterModel, models, status, meteo, constants=nothing, extra=nothing) + status.soil_water_content = model.baseline - 0.001 * meteo.T + return nothing +end + +struct DocDomainSoilEvaporationModel <: AbstractDoc_Domain_Soil_EvaporationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DocDomainSoilEvaporationModel) = (soil_water_content=0.0,) +PlantSimEngine.outputs_(::DocDomainSoilEvaporationModel) = (evaporation=0.0,) +PlantSimEngine.meteo_inputs_(::DocDomainSoilEvaporationModel) = (T=0.0,) + +function PlantSimEngine.run!(model::DocDomainSoilEvaporationModel, models, status, meteo, constants=nothing, extra=nothing) + status.evaporation = model.coefficient * status.soil_water_content * meteo.T + return nothing +end + +struct DocDomainSceneEvapotranspirationModel <: AbstractDoc_Domain_Scene_EvapotranspirationModel end + +PlantSimEngine.inputs_(::DocDomainSceneEvapotranspirationModel) = NamedTuple() +PlantSimEngine.outputs_(::DocDomainSceneEvapotranspirationModel) = (evapotranspiration=0.0,) + +PlantSimEngine.dep(::DocDomainSceneEvapotranspirationModel) = ( + plant_transpiration=AllDomains(kind=:plant, process=:doc_domain_plant_transpiration, var=:transpiration, policy=Integrate()), + soil_evaporation=AllDomains(kind=:soil, process=:doc_domain_soil_evaporation, var=:evaporation, policy=Integrate()), +) + +function PlantSimEngine.run!(::DocDomainSceneEvapotranspirationModel, models, status, meteo, constants=nothing, extra=nothing) + plant_values = dependency_values(extra, :plant_transpiration) + soil_values = dependency_values(extra, :soil_evaporation) + status.evapotranspiration = + sum(filter(x -> !isnothing(x), plant_values)) + + sum(filter(x -> !isnothing(x), soil_values)) + return nothing +end +nothing +``` + +Now each domain can be built independently. The oil palm and maize mappings use +the same model types but different parameters. In a real application these +could be completely different plant models, with different processes and +different internal mappings. + +```@example domains +oil_palm_mapping = ModelMapping( + ModelSpec(DocDomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Hour(1)), + ModelSpec(DocDomainPlantTranspirationModel(0.01)) |> TimeStepModel(Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), +) + +maize_mapping = ModelMapping( + ModelSpec(DocDomainAbsorbedRadiationModel(0.3)) |> TimeStepModel(Hour(1)), + ModelSpec(DocDomainPlantTranspirationModel(0.02)) |> TimeStepModel(Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), +) + +soil_mapping = ModelMapping( + ModelSpec(DocDomainSoilWaterModel(0.35)) |> TimeStepModel(Hour(1)), + ModelSpec(DocDomainSoilEvaporationModel(0.2)) |> TimeStepModel(Hour(1)), + status=(soil_water_content=0.0, evaporation=0.0), +) + +scene_mapping = ModelMapping( + ModelSpec(DocDomainSceneEvapotranspirationModel()) |> TimeStepModel(Day(1)), + status=(evapotranspiration=0.0,), +) + +simulation_mapping = SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:maize, maize_mapping; kind=:plant), + Domain(:soil, soil_mapping; kind=:soil), + Domain(:scene, scene_mapping; kind=:scene), +) +nothing +``` + +The meteorology is hourly. The plant and soil models run hourly, while the +scene model runs every day and integrates the producer streams. + +```@example domains +hourly_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Hour(1)) + for _ in 1:25 +]) + +sim = run!(simulation_mapping, hourly_meteo; check=true) +round(status(sim, :scene).evapotranspiration; digits=2) +``` + +The final value is the daily integral of the two plant transpiration streams and +the soil evaporation stream. + +## Inspecting the compiled simulation + +Domain simulations expose small structured explanation helpers. These are meant +for users and for agents that need to repair a mapping from concrete symbols. + +```@example domains +sort([(row.domain, row.kind) for row in explain_domains(sim)]) +``` + +```@example domains +sort([(row.domain, row.process, row.dt_seconds) for row in explain_schedule(sim)]) +``` + +```@example domains +sort( + [(row.dependency, string(row.producer), row.policy) for row in explain_domain_dependencies(sim)]; + by=string, +) +``` + +The model-level explanation also includes the weather variables declared with +`meteo_inputs_` and any mutable environment variables declared with +`meteo_outputs_`: + +```@example domains +[(row.domain, row.process, collect(keys(row.meteo_inputs))) for row in explain_domain_models(sim) if !isempty(row.meteo_inputs)] +``` + +## Explicit routes + +`AllDomains(...)` value dependencies are the most direct way for a scene model +to consume several domain outputs. Routes are another option when you want to +materialize values into the target status before the target model runs. + +```julia +Route( + from=AllDomains(kind=:plant, process=:plant_transpiration, var=:transpiration), + to=DomainRouteTarget(:scene, var=:plant_transpirations, process=:scene_evapotranspiration), + cardinality=ManyToOneVector(), +) +``` + +Use `ManyToOneVector()` when the target model needs one value per matching +producer, and `ManyToOneAggregate(f)` when it needs a scalar reduction. For an +MTG-backed target domain, `OneToManyBroadcast()` can broadcast one source value +into every status at the target scale before that domain runs. + +When a route targets a single-status domain variable consumed by one target +model, the target status slot is created from that model's `inputs_` default if +the user did not initialize it explicitly. Variables that are only route +materialization slots and are not model inputs still need to be initialized in +the target status. + +When graph-domain values are aggregated across time, PlantSimEngine aligns them +by MTG node id. Growth and pruning inside the aggregation window therefore do +not require every timestep to publish vectors with the same length. + +## Hard-domain dependencies + +Use `HardDomains(...)` when a model must manually run selected producer +models, for example to control an iterative energy-balance solver: + +```julia +PlantSimEngine.dep(::SceneEnergyBalanceModel) = ( + leaf_energy=HardDomains(kind=:plant, scale=:Leaf, process=:leaf_energy_balance), +) + +function PlantSimEngine.run!(::SceneEnergyBalanceModel, models, status, meteo, constants=nothing, extra=nothing) + leaf_targets = dependency_targets(extra, :leaf_energy) + for iteration in 1:10 + for target in leaf_targets + run_target!(target) + end + converged(status) && break + end + return nothing +end +``` + +Each target represents one selected model on one status. For an MTG-backed +producer domain, this means one target per matching node status, such as one +target per leaf. `run_target!` mutates that status like a normal model call, +but it does not append to domain streams or outputs unless `publish=true` is +requested. Trial iterations should usually run with `publish=false`, then the +final accepted hard-dependency call can use `publish=true`. + +For same-status hard dependencies, the same target API can replace direct model +calls: + +```julia +function PlantSimEngine.run!(::PhyllochronModel, models, status, meteo, constants, extra) + run_target!(models, status, :phytomer_emission; meteo=meteo, constants=constants, extra=extra) + return nothing +end +``` + +## MTG-backed plant domains + +The same `Domain` wrapper can be used around a scale-keyed `ModelMapping`. +When running on an MTG, the `selector` decides which subtree roots belong to +the domain: + +```julia +oil_palm = Domain(:oil_palm, xpalm_mapping; kind=:plant, selector=node -> node[:species] == :oil_palm) +maize = Domain(:maize, maize_mapping; kind=:plant, selector=node -> node[:species] == :maize) +sim = run!(scene_mtg, SimulationMapping(oil_palm, maize, scene), meteo) +``` + +Status inspection keeps both the domain-local view and the global scale view: + +```julia +status(sim, :oil_palm, :Leaf) # only oil palm leaves +status(sim, :maize, :Leaf) # only maize leaves +status(sim, :Leaf) # all leaves across graph-backed domains +``` + +If a declared graph scale currently has no active statuses, for example after +pruning all leaves, these status queries return `Status[]`. The same empty scale +is reported by `explain_domain_statuses(sim)` with `nstatuses=0`. + +Selectors can also match several non-overlapping roots, for example +`selector=:Plant` to run one mapping over every plant subtree. Selectors that +match overlapping roots are rejected because they would register the same organ +twice. + +## Meteorology and microclimate + +Plain meteorology passed to `run!(SimulationMapping, meteo)` is wrapped as a +`GlobalConstant` environment backend. This preserves the existing behavior: +every object sees the same weather row at a given timestep. +If a model declares `meteo_inputs_`, running without meteorology now fails +during validation with the missing environment variables, instead of failing +later inside the model code. + +Spatial or mutable microclimate should live in a specialized backend, not in +PlantSimEngine itself. A backend subtypes `AbstractEnvironmentBackend` and +implements the sampling and update hooks: + +```julia +PlantSimEngine.sample(backend, variable, support, time) +PlantSimEngine.scatter!(backend, variable, support, value, time) +PlantSimEngine.update_index!(backend, entities) +``` + +Models declare which environment variables they read and write: + +```julia +PlantSimEngine.meteo_inputs_(::LeafEnergyBalanceModel) = ( + T=0.0, + Rh=0.0, + Wind=0.0, + Ri_PAR_f=0.0, + CO2=400.0, +) + +PlantSimEngine.meteo_outputs_(::MicroclimateUpdateModel) = (T=0.0,) +``` + +`EnvironmentSupport(domain, scale, process, status)` is passed to the backend, +so an octree, voxel, layered-canopy, or CFD backend can decide how to sample the +environment for the current organ or plant. In graph-backed domains, `status` +is the current node status. + +## Updating a variable + +Duplicate canonical writers are still errors by default. If a model is meant to +update a variable already produced at the same scale, declare that scenario rule +on the updating `ModelSpec`: + +```julia +ModelSpec(LeafPruningModel()) |> + Updates(:leaf_biomass; after=:carbon_allocation) +``` + +Only variables with several writers need an `Updates(...)` declaration. The +declaration adds an ordering edge and downstream consumers infer the terminal +updater as the effective source. + +## Growth and topology changes + +MTG-backed domain models receive the underlying `GraphSimulation` as `extra`, +so existing growth models can keep using the MTG registration APIs: + +```julia +add_organ!(parent_node, extra, "+", :Leaf, 2; check=true) +remove_organ!(leaf_node, extra) +remove_organ!(internode_node, extra; recursive=true) +reparent_organ!(leaf_node, new_parent_node, extra) +``` + +These helpers update statuses, multiscale `RefVector`s, temporal streams, and +domain outputs. `update_index!(backend, entities)` is called after each domain +step so spatial environment backends can refresh their entity index after +growth, pruning, or geometry changes. + +## Current boundaries + +Cross-domain dependencies are explicit. PlantSimEngine does not infer them from +matching variable names across domains. + +The domain layer defines the environment backend protocol, but it does not +implement an octree, voxel grid, or energy-balance solver. Those belong in +domain packages that provide models or environment backends. + +Scene domains run after plant and soil domains in the current acyclic runner. +Routes that feed an MTG-backed target domain therefore need their source domain +to run earlier in the same timestep. diff --git a/docs/src/model_traits.md b/docs/src/model_traits.md index 6c9633451..c8fd7bc12 100644 --- a/docs/src/model_traits.md +++ b/docs/src/model_traits.md @@ -74,7 +74,7 @@ Supported forms include: - range: `(Dates.Minute(30), Dates.Hour(2))`; - named tuple: `(; required=..., preferred=...)`. -`required` is enforced when runtime uses meteo-derived timestep. +`required` is enforced when runtime uses meteo-derived timestep. `preferred` is informational only. ### `meteo_hint(::Type{<:MyModel})` @@ -98,6 +98,42 @@ Where: - `bindings` is compatible with `MeteoBindings(...)`, - `window` is compatible with `MeteoWindow(...)`. +### `meteo_inputs_(::MyModel)` +### `meteo_outputs_(::MyModel)` + +Declare meteorology or microclimate variables separately from object status +variables. + +Default: + +```julia +PlantSimEngine.meteo_inputs_(::AbstractModel) = NamedTuple() +PlantSimEngine.meteo_outputs_(::AbstractModel) = NamedTuple() +``` + +Use `meteo_inputs_` for variables read from the weather or environment backend: + +```julia +PlantSimEngine.meteo_inputs_(::LeafEnergyBalanceModel) = ( + T=0.0, + Rh=0.0, + Wind=0.0, + Ri_PAR_f=0.0, + CO2=400.0, +) +``` + +Use `meteo_outputs_` when a model updates a mutable environment backend, for +example a microclimate model updating local air temperature: + +```julia +PlantSimEngine.outputs_(::MicroclimateUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_outputs_(::MicroclimateUpdateModel) = (T=0.0,) +``` + +The current runtime scatters `meteo_outputs_` from status variables, so the +same variable should usually be declared in `outputs_` as well. + ### `TimeStepDependencyTrait(::Type{<:MyModel})` ### `ObjectDependencyTrait(::Type{<:MyModel})` @@ -127,6 +163,8 @@ For model-level traits, the documented set is now: - `output_policy`, - `timestep_hint`, - `meteo_hint`, +- `meteo_inputs_`, +- `meteo_outputs_`, - `TimeStepDependencyTrait`, - `ObjectDependencyTrait`. diff --git a/docs/src/multiscale/multiscale_considerations.md b/docs/src/multiscale/multiscale_considerations.md index 091cf32e6..226ff0c96 100644 --- a/docs/src/multiscale/multiscale_considerations.md +++ b/docs/src/multiscale/multiscale_considerations.md @@ -142,7 +142,7 @@ Converting to a dictionary of DataFrame objects can make such queries easier to !!! warning Currently, the `:node` entry only shallow copies nodes. The `:node` values at each scale for every timestep actually reflect the final state of the node, meaning attribute values may not correspond to the value at that timestep. You may need to output these values via a dedicated model to keep track of them properly. - Also note that there currently is no way of removing nodes. Nodes corresponding to organs considered to be pruned/dead/aborted are still present in the output data structure. + Nodes can be removed from an active graph simulation with [`remove_organ!`](@ref), but already-exported outputs are historical records. A pruned/dead/aborted organ may still appear in past output rows, while current statuses and future outputs no longer include it. Multi-scale simulations, especially for plants which have thousands of leaves, internodes, root branches, buds and fruits, may compute huge amounts of data. Just like in single-scale simulations, it is possible to keep only variables whose values you want to track for every timestep, and filter the rest out, using the `tracked_outputs` keyword argument for the [`run!`](@ref) function. diff --git a/docs/src/step_by_step/advanced_coupling.md b/docs/src/step_by_step/advanced_coupling.md index 07b81c8bb..9fc612cff 100644 --- a/docs/src/step_by_step/advanced_coupling.md +++ b/docs/src/step_by_step/advanced_coupling.md @@ -33,7 +33,7 @@ When run, `Process2Model` calls another process's [`run!`](@ref) function explic ```julia function PlantSimEngine.run!(::Process2Model, models, status, meteo, constants, extra) # computing var3 using process1: - run!(models.process1, models, status, meteo, constants) + run_target!(models, status, :process1; meteo=meteo, constants=constants, extra=extra) # computing var4 and var5: status.var4 = status.var3 * 2.0 status.var5 = status.var4 + 1.0 * meteo.T + 2.0 * meteo.Wind + 3.0 * meteo.Rh @@ -61,4 +61,4 @@ PlantSimEngine.dep(::Process2Model) = (process1=Process1Model,) ## Examples in the wild -You can find a typical example in a companion package: [PlantBioPhysics.jl](https://github.com/VEZY/PlantBiophysics.jl). An energy balance model, the [Monteith model](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl), needs to [iteratively run a photosynthesis model](https://github.com/VEZY/PlantBiophysics.jl/blob/c1a75f294109d52dc619f764ce51c6ca1ea897e8/src/processes/energy/Monteith.jl#L154) in its [`run!`](@ref) function. \ No newline at end of file +You can find a typical example in a companion package: [PlantBioPhysics.jl](https://github.com/VEZY/PlantBiophysics.jl). An energy balance model, the [Monteith model](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl), needs to [iteratively run a photosynthesis model](https://github.com/VEZY/PlantBiophysics.jl/blob/c1a75f294109d52dc619f764ce51c6ca1ea897e8/src/processes/energy/Monteith.jl#L154) in its [`run!`](@ref) function. diff --git a/docs/src/working_with_data/inputs.md b/docs/src/working_with_data/inputs.md index 3c94c985b..d2540035b 100644 --- a/docs/src/working_with_data/inputs.md +++ b/docs/src/working_with_data/inputs.md @@ -30,7 +30,7 @@ models = ModelMapping( ) ``` -For single-scale mappings, `type_promotion` is applied while the backing status is constructed. It follows the same semantics as the deprecated [`ModelList`](@ref): model-provided default values are converted, while values explicitly passed in `status` keep the type chosen by the user. If those values should also be `Float32`, pass them as `Float32` values directly. +For single-scale mappings, `type_promotion` is applied while the backing status is constructed: model-provided default values are converted, while values explicitly passed in `status` keep the type chosen by the user. If those values should also be `Float32`, pass them as `Float32` values directly. For multiscale mappings, the per-node statuses do not exist when [`ModelMapping`](@ref) is constructed. The promotion map is stored on the mapping and applied when the MTG simulation is initialized: diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantHelpers.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantHelpers.jl new file mode 100644 index 000000000..d8c36fdd7 --- /dev/null +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantHelpers.jl @@ -0,0 +1,14 @@ +function get_root_end_node(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root, filter_fun=MultiScaleTreeGraph.isleaf) +end + +function get_roots_count(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root)) +end + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x -> 1, symbol=:Leaf)) +end diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl index 15b3ec356..d6e713c22 100644 --- a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl @@ -6,21 +6,7 @@ # But it should illustrate the basics of simulating a growing multiscale plant with PlantSimEngine's model approach ########################################### -function get_root_end_node(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - return MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root, filter_fun=MultiScaleTreeGraph.isleaf) -end - -function get_roots_count(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - return length(MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root)) -end - -function get_n_leaves(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - nleaves = length(MultiScaleTreeGraph.traverse(root, x -> 1, symbol=:Leaf)) - return nleaves -end +include(joinpath(@__DIR__, "ToyPlantHelpers.jl")) PlantSimEngine.@process "organ_emergence" verbose = false diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl index bdd4071b5..6c305cd5b 100644 --- a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl @@ -6,21 +6,7 @@ # But it should illustrate the basics of simulating a growing multiscale plant with PlantSimEngine's model approach ########################################### -function get_root_end_node(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - return MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root, filter_fun=MultiScaleTreeGraph.isleaf) -end - -function get_roots_count(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - return length(MultiScaleTreeGraph.traverse(root, x -> x, symbol=:Root)) -end - -function get_n_leaves(node::MultiScaleTreeGraph.Node) - root = MultiScaleTreeGraph.get_root(node) - nleaves = length(MultiScaleTreeGraph.traverse(root, x -> 1, symbol=:Leaf)) - return nleaves -end +include(joinpath(@__DIR__, "ToyPlantHelpers.jl")) PlantSimEngine.@process "organ_emergence" verbose = false diff --git a/examples/dummy.jl b/examples/dummy.jl index f3a556f1d..82dcde9f1 100644 --- a/examples/dummy.jl +++ b/examples/dummy.jl @@ -37,7 +37,7 @@ PlantSimEngine.outputs_(::Process2Model) = (var4=-Inf, var5=-Inf) PlantSimEngine.dep(::Process2Model) = (process1=AbstractProcess1Model,) function PlantSimEngine.run!(::Process2Model, models, status, meteo, constants=nothing, extra=nothing) # computing var3 using process1: - PlantSimEngine.run!(models.process1, models, status, meteo, constants) + PlantSimEngine.run_target!(models, status, :process1; meteo=meteo, constants=constants, extra=extra) # computing var4 and var5: status.var4 = status.var3 * 2.0 status.var5 = status.var4 + 1.0 * meteo.T + 2.0 * meteo.Wind + 3.0 * meteo.Rh @@ -63,7 +63,7 @@ PlantSimEngine.outputs_(::Process3Model) = (var4=-Inf, var6=-Inf,) PlantSimEngine.dep(::Process3Model) = (process2=Process2Model,) function PlantSimEngine.run!(::Process3Model, models, status, meteo, constants=nothing, extra=nothing) # computing var3 using process1: - PlantSimEngine.run!(models.process2, models, status, meteo, constants, extra) + PlantSimEngine.run_target!(models, status, :process2; meteo=meteo, constants=constants, extra=extra) # re-computing var4: status.var4 = status.var4 * 2.0 status.var6 = status.var5 + status.var4 diff --git a/examples/maespa_domain_example.jl b/examples/maespa_domain_example.jl new file mode 100644 index 000000000..e9822168e --- /dev/null +++ b/examples/maespa_domain_example.jl @@ -0,0 +1,491 @@ +using Dates +using PlantMeteo +using PlantSimEngine +using MultiScaleTreeGraph + +include(joinpath(@__DIR__, "plantbiophysics_subsample", "Tuzet.jl")) +include(joinpath(@__DIR__, "plantbiophysics_subsample", "FvCB.jl")) +include(joinpath(@__DIR__, "plantbiophysics_subsample", "Monteith.jl")) + +PlantSimEngine.@process "soil_water" verbose = false +PlantSimEngine.@process "scene_eb" verbose = false +PlantSimEngine.@process "leaf_state" verbose = false +PlantSimEngine.@process "lai_dynamic" verbose = false +PlantSimEngine.@process "alloc_a" verbose = false +PlantSimEngine.@process "alloc_b" verbose = false + +sat_vp_kpa(T) = 0.6108 * exp(17.27 * T / (T + 237.3)) +vpd_kpa(meteo) = max(0.02, sat_vp_kpa(meteo.T) * (1.0 - meteo.Rh)) +rh_from_vpd(T, vpd) = clamp(1.0 - vpd / sat_vp_kpa(T), 0.05, 0.99) +duration_seconds(meteo) = Dates.value(Dates.Millisecond(meteo.duration)) / 1000.0 +e_sat_kpa(T) = PlantMeteo.e_sat(T) + +struct SoilWater{T} <: AbstractSoil_WaterModel + theta_sat::T + psi_e::T + b::T + depth1::T + depth2::T +end + +PlantSimEngine.inputs_(::SoilWater) = (transpiration=0.0, infiltration=0.0) +PlantSimEngine.outputs_(::SoilWater) = (theta1=0.32, theta2=0.34, psi_soil=-0.1) + +function PlantSimEngine.run!(m::SoilWater, models, status, meteo, constants, extra=nothing) + withdrawal = max(status.transpiration, 0.0) + recharge = max(status.infiltration, 0.0) + status.theta1 = clamp(status.theta1 + (recharge - 0.7 * withdrawal) / max(m.depth1 * 1000.0, 1.0), 0.04, m.theta_sat) + status.theta2 = clamp(status.theta2 - 0.3 * withdrawal / max(m.depth2 * 1000.0, 1.0), 0.04, m.theta_sat) + rel = clamp(status.theta1 / m.theta_sat, 0.05, 1.0) + status.psi_soil = m.psi_e * rel^(-m.b) + return nothing +end + +struct LeafState <: AbstractLeaf_StateModel end + +PlantSimEngine.inputs_(::LeafState) = NamedTuple() +PlantSimEngine.outputs_(::LeafState) = (leaf_area=0.0, leaf_carbon=0.0) + +PlantSimEngine.run!(::LeafState, models, status, meteo, constants, extra=nothing) = nothing + +""" + LAIModel(area) + +Compute scene leaf area and leaf area index from all leaves routed into the +scene domain. +""" +struct LAIModel{T} <: AbstractLai_DynamicModel + area::T + + function LAIModel(area::T) where {T} + area > 0 || throw(ArgumentError("`area` must be strictly positive.")) + new{T}(area) + end +end + +PlantSimEngine.inputs_(::LAIModel) = (leaf_areas=[-Inf],) +PlantSimEngine.outputs_(::LAIModel) = (lai=0.0, leaf_area=-Inf) + +function PlantSimEngine.run!(m::LAIModel, models, status, meteo, constants, extra=nothing) + status.leaf_area = sum(status.leaf_areas) + status.lai = status.leaf_area / m.area + return nothing +end + +struct SceneEB{I,T} <: AbstractScene_EbModel + maxiter::I + tol_t::T + tol_vpd::T + tree_height::T + zht::T + zpd::T + z0ht::T + ground_area::T + qc::T + gbcan_min::T + von_karman::T +end + +function SceneEB( + maxiter, + tol_t, + tol_vpd; + tree_height=2.0, + zht=4.0, + zpd=0.75 * tree_height, + z0ht=0.1 * tree_height, + ground_area=1.0, + qc=0.0, + gbcan_min=0.0123, + von_karman=0.41, +) + ground_area > 0.0 || throw(ArgumentError("`ground_area` must be strictly positive.")) + tree_height > 0.0 || throw(ArgumentError("`tree_height` must be strictly positive.")) + zht > 0.0 || throw(ArgumentError("`zht` must be strictly positive.")) + return SceneEB( + maxiter, + promote(tol_t, + tol_vpd, + tree_height, + zht, + zpd, + z0ht, + ground_area, + qc, + gbcan_min, + von_karman)... + ) +end + +PlantSimEngine.inputs_(::SceneEB) = (lai=0.0, leaf_area=0.0) +PlantSimEngine.outputs_(::SceneEB) = ( + canopy_tair=20.0, + canopy_vpd=1.0, + canopy_rh=0.7, + canopy_htot=0.0, + canopy_gcanop=0.0, + scene_transpiration=0.0, + scene_assimilation=0.0, + psi_soil=-0.1, + iterations=0, +) + +struct SceneEBSolverResult + tair::Float64 + vpd::Float64 + rh::Float64 + psi_soil::Float64 + final_meteo + iterations::Int + htot::Float64 + gcanop::Float64 + lai::Float64 +end + +function _scene_leaf_meteo(meteo, tair_canopy, vpd_canopy) + return Atmosphere( + T=tair_canopy, + Rh=rh_from_vpd(tair_canopy, vpd_canopy), + Wind=meteo.Wind, + P=meteo.P, + Cₐ=meteo.Cₐ, + Ri_PAR_f=meteo.Ri_PAR_f, + Ri_SW_f=meteo.Ri_SW_f, + duration=meteo.duration, + ) +end + +function _prepare_scene_leaf_target!(target, meteo, tair_canopy, vpd_canopy, psi_soil) + target.status.Ra_SW_f = meteo.Ri_SW_f + target.status.aPPFD = meteo.Ri_PAR_f + target.status.Ψₗ = psi_soil + return nothing +end + +function _run_scene_leaf_targets!(leaf_targets, meteo, tair_canopy, vpd_canopy, psi_soil, ground_area; publish=false) + total_rn = 0.0 + total_lambda_e = 0.0 + total_h = 0.0 + total_a = 0.0 + local_meteo = _scene_leaf_meteo(meteo, tair_canopy, vpd_canopy) + for target in leaf_targets + _prepare_scene_leaf_target!(target, meteo, tair_canopy, vpd_canopy, psi_soil) + run_call!(target; meteo=local_meteo, publish=publish) # Publish is false because we iterate here and only want to publish the final solution at the end + leaf_area = target.status.leaf_area + total_rn += target.status.Rn * leaf_area + total_lambda_e += target.status.λE * leaf_area + total_h += target.status.H * leaf_area + total_a += target.status.A * leaf_area + end + return ( + rn=total_rn / ground_area, + lambda_e=total_lambda_e / ground_area, + h=total_h / ground_area, + a=total_a / ground_area, + meteo=local_meteo, + ) +end + +function gbcanms(wind, zht, tree_height; gbcan_min=0.0123, von_karman=0.41) + zpd = 0.75 * tree_height + z0 = 0.1 * tree_height + zstar = max(zht, eps(Float64)) + wind2 = max(wind, 1.0e-6) + + if zstar <= tree_height + wind2 *= exp(0.13155 * (tree_height / zstar - 1.0)) + zstar = 2.0 * tree_height + end + + zstar = max(zstar, zpd + z0 + 1.0e-6) + windstar = wind2 * von_karman / log((zstar - zpd) / z0) + alpha1 = 1.5 + zw = zpd + alpha1 * (tree_height - zpd) + gbcanmsini = windstar * von_karman / log((zstar - zpd) / (zw - zpd)) + gbcanmsrou = windstar * von_karman / ((zw - tree_height) / (zw - zpd)) + canopy_air_ms = max(1.0 / (1.0 / gbcanmsini + 1.0 / gbcanmsrou), gbcan_min) + + alpha = 2.0 + z0ht2 = 0.01 + kh = alpha1 * von_karman * windstar * (tree_height - zpd) + soil_denominator = tree_height * exp(alpha) * + (exp(-alpha * z0ht2 / tree_height) - exp(-alpha * (zpd + z0) / tree_height)) + soil_canopy_ms = max(alpha * kh / soil_denominator, 0.0) + return (canopy_air_ms=canopy_air_ms, soil_canopy_ms=soil_canopy_ms) +end + +function tvpdcanopcalc(m::SceneEB, fluxes, meteo_above, canopy_meteo, constants) + gbs = gbcanms( + meteo_above.Wind, + m.zht, + m.tree_height; + gbcan_min=m.gbcan_min, + von_karman=m.von_karman, + ) + gbcan_ms = gbs.canopy_air_ms + tair_above = meteo_above.T + vpd_above = max(0.01, meteo_above.VPD) + qn = fluxes.rn + qe = fluxes.lambda_e + rad_interc = get(fluxes, :rad_interc, 0.0) + rnettot = qn + rad_interc + etot = qe + htot = rnettot - etot - m.qc + heat_conductance = constants.Cₚ * canopy_meteo.ρ * gbcan_ms + + tair_new = tair_above + htot / heat_conductance + tair_new = clamp(tair_new, tair_above - 10.0, tair_above + 10.0) + + vpair_above = e_sat_kpa(tair_above) - vpd_above + vpair_canopy = vpair_above + etot * canopy_meteo.γ / heat_conductance + vpd_new = max(0.01, e_sat_kpa(tair_new) - vpair_canopy) + vpd_new = clamp(vpd_new, max(0.01, vpd_above - 1.5), vpd_above + 1.5) + rh_new = clamp(1.0 - vpd_new / e_sat_kpa(tair_new), 0.0, 1.0) + return (tair=tair_new, vpd=vpd_new, rh=rh_new, htot=htot, gcanop=gbcan_ms) +end + +function _solve_scene_energy_balance!(m::SceneEB, leaf_targets, soil_target, status, meteo, constants=PlantMeteo.Constants()) + tair_above = meteo.T + vpd_above = max(0.01, meteo.VPD) + tair_canopy = tair_above + vpd_canopy = vpd_above + psi_soil = soil_target.status.psi_soil + final_meteo = meteo + last_update = (tair=tair_canopy, vpd=vpd_canopy, rh=meteo.Rh, htot=0.0, gcanop=0.0) + + for iter in 1:m.maxiter + # Run the energy balance of each leaf, and aggregate the fluxes at the canopy scale: + fluxes = _run_scene_leaf_targets!(leaf_targets, meteo, tair_canopy, vpd_canopy, psi_soil, m.ground_area) + fluxes = merge(fluxes, (lai=status.lai, rad_interc=0.0)) + # Update the canopy-scale meteo based on the leaf fluxes, and check for convergence: + final_meteo = fluxes.meteo + update = tvpdcanopcalc(m, fluxes, meteo, final_meteo, constants) + last_update = update + if abs(update.tair - tair_canopy) < m.tol_t && abs(update.vpd - vpd_canopy) < m.tol_vpd + tair_canopy = update.tair + vpd_canopy = update.vpd + return SceneEBSolverResult( + tair_canopy, + vpd_canopy, + update.rh, + psi_soil, + _scene_leaf_meteo(meteo, tair_canopy, vpd_canopy), + iter, + update.htot, + update.gcanop, + status.lai, + ) + end + tair_canopy = 0.5 * (tair_canopy + update.tair) # take the average to help convergence + vpd_canopy = 0.5 * (vpd_canopy + update.vpd) + end + + error( + "SceneEB did not converge after $(m.maxiter) iterations ", + "(tol_t=$(m.tol_t), tol_vpd=$(m.tol_vpd), ", + "last_tair=$(last_update.tair), last_vpd=$(last_update.vpd))." + ) +end + +function _publish_scene_leaf_solution!(leaf_targets, solution::SceneEBSolverResult, meteo, ground_area) + fluxes = _run_scene_leaf_targets!( + leaf_targets, + meteo, + solution.tair, + solution.vpd, + solution.psi_soil, + ground_area; + publish=true, + ) + for target in leaf_targets + target.status.leaf_carbon += target.status.A * target.status.leaf_area * duration_seconds(meteo) * 12.0e-6 + end + return fluxes +end + +function _run_scene_soil_feedback!(soil_target, transpiration_mm) + soil_target.status.transpiration = transpiration_mm + soil_target.status.infiltration = 0.0 + run_call!(soil_target; publish=true) + return soil_target.status.psi_soil +end + +function PlantSimEngine.run!(m::SceneEB, models, status, meteo, constants, extra) + leaf_targets = dependency_targets(extra, :energy_balance) + soil_target = only(dependency_targets(extra, :soil)) + solution = _solve_scene_energy_balance!(m, leaf_targets, soil_target, status, meteo, constants) + fluxes = _publish_scene_leaf_solution!(leaf_targets, solution, meteo, m.ground_area) + transpiration_mm = λE_to_E(fluxes.lambda_e, solution.final_meteo.λ) * duration_seconds(meteo) * 18.0e-6 + psi_soil = _run_scene_soil_feedback!(soil_target, transpiration_mm) + + status.canopy_tair = solution.tair + status.canopy_vpd = solution.vpd + status.canopy_rh = solution.rh + status.canopy_htot = solution.htot + status.canopy_gcanop = solution.gcanop + status.scene_transpiration = transpiration_mm + status.scene_assimilation = fluxes.a + status.psi_soil = psi_soil + status.iterations = solution.iterations + return nothing +end + +alloc_inputs() = (leaf_carbon=[0.0],) +alloc_outputs() = (daily_growth=0.0, leaf_pool=0.0, wood_pool=0.0) + +function allocate!(status, leaf_fraction, wood_fraction) + carbon = sum(status.leaf_carbon) + status.daily_growth = carbon + status.leaf_pool += leaf_fraction * carbon + status.wood_pool += wood_fraction * carbon + return nothing +end + +struct AllocA <: AbstractAlloc_AModel + leaf_fraction::Float64 + wood_fraction::Float64 +end + +struct AllocB <: AbstractAlloc_BModel + leaf_fraction::Float64 + wood_fraction::Float64 +end + +PlantSimEngine.inputs_(::AllocA) = alloc_inputs() +PlantSimEngine.outputs_(::AllocA) = alloc_outputs() +PlantSimEngine.inputs_(::AllocB) = alloc_inputs() +PlantSimEngine.outputs_(::AllocB) = alloc_outputs() + +PlantSimEngine.run!(m::AllocA, models, status, meteo, constants, extra=nothing) = + allocate!(status, m.leaf_fraction, m.wood_fraction) +PlantSimEngine.run!(m::AllocB, models, status, meteo, constants, extra=nothing) = + allocate!(status, m.leaf_fraction, m.wood_fraction) + +function build_maespa_scene() + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant_a = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + plant_a[:species] = :A + axis_a = Node(plant_a, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + for i in 1:2 + leaf = Node(axis_a, MultiScaleTreeGraph.NodeMTG("+", :Leaf, i, 3)) + leaf[:species] = :A + end + + plant_b = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 2, 1)) + plant_b[:species] = :B + axis_b = Node(plant_b, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + for i in 1:3 + leaf = Node(axis_b, MultiScaleTreeGraph.NodeMTG("+", :Leaf, i, 3)) + leaf[:species] = :B + end + return scene +end + +has_species(node, species) = + try + node[:species] == species + catch + false + end + +function maespa_mapping(; scene_model=SceneEB(25, 0.03, 0.005)) + leaf_a = ( + ModelSpec(Monteith(; ε=0.955, maxiter=20, ΔT=0.02)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(Fvcb(; VcMaxRef=72.0, JMaxRef=135.0, RdRef=1.1)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(Tuzet(; g0=0.015, g1=4.8, Ψᵥ=-1.4, sf=3.2, Γ=42.0)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(LeafState()) |> TimeStepModel(Dates.Hour(1)), + Status(Ra_SW_f=0.0, sky_fraction=1.0, d=0.035, aPPFD=0.0, Ψₗ=-0.1, leaf_area=0.018, leaf_carbon=0.0), + ) + leaf_b = ( + ModelSpec(Monteith(; ε=0.955, maxiter=20, ΔT=0.02)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(Fvcb(; VcMaxRef=58.0, JMaxRef=110.0, RdRef=1.3)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(Tuzet(; g0=0.012, g1=3.5, Ψᵥ=-1.1, sf=3.8, Γ=42.0)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(LeafState()) |> TimeStepModel(Dates.Hour(1)), + Status(Ra_SW_f=0.0, sky_fraction=0.8, d=0.028, aPPFD=0.0, Ψₗ=-0.1, leaf_area=0.014, leaf_carbon=0.0), + ) + + plant_a = ModelMapping( + :Plant => ( + ModelSpec(AllocA(0.35, 0.55)) |> + MultiScaleModel([:leaf_carbon => [:Leaf => :leaf_carbon]]) |> + TimeStepModel(ClockSpec(24.0, 0.0)), + Status(leaf_pool=0.0, wood_pool=0.0), + ), + :Leaf => leaf_a, + ) + plant_b = ModelMapping( + :Plant => ( + ModelSpec(AllocB(0.55, 0.35)) |> + MultiScaleModel([:leaf_carbon => [:Leaf => :leaf_carbon]]) |> + TimeStepModel(ClockSpec(24.0, 0.0)), + Status(leaf_pool=0.0, wood_pool=0.0), + ), + :Leaf => leaf_b, + ) + soil = ModelMapping( + ModelSpec(SoilWater(0.45, -0.03, 4.4, 0.25, 0.75)) |> TimeStepModel(Dates.Hour(1)), + status=(theta1=0.33, theta2=0.36, psi_soil=-0.10, transpiration=0.0, infiltration=0.0), + ) + ground_area = scene_model.ground_area + scene = ModelMapping( + ModelSpec(LAIModel(ground_area)) |> + Inputs(:leaf_areas => Many(kind=:plant, scale=:Leaf, process=:leaf_state, var=:leaf_area)) |> + TimeStep(Dates.Day(1)), + ModelSpec(scene_model) |> + Calls( + :energy_balance => Many(kind=:plant, scale=:Leaf, process=:energy_balance), + :soil => One(kind=:soil, process=:soil_water), + ) |> + TimeStep(Dates.Hour(1)), + status=( + leaf_area=0.0, + lai=0.0, + canopy_tair=20.0, + canopy_vpd=1.0, + canopy_rh=0.7, + canopy_htot=0.0, + canopy_gcanop=0.0, + scene_transpiration=0.0, + scene_assimilation=0.0, + psi_soil=-0.1, + iterations=0, + ), + ) + return SimulationMapping( + Domain(:plant_A, plant_a; kind=:plant, selector=node -> MultiScaleTreeGraph.symbol(node) == :Plant && has_species(node, :A)), + Domain(:plant_B, plant_b; kind=:plant, selector=node -> MultiScaleTreeGraph.symbol(node) == :Plant && has_species(node, :B)), + Domain(:soil, soil; kind=:soil), + Domain(:scene, scene; kind=:scene), + ) +end + +function maespa_meteo(; nhours=24) + return Weather([ + Atmosphere( + T=22.0 + 5.0 * sinpi((hour - 7) / 12), + Rh=clamp(0.72 - 0.22 * sinpi((hour - 7) / 12), 0.35, 0.90), + Wind=1.2 + 0.3 * sinpi(hour / 12), + Ri_PAR_f=max(0.0, 900.0 * sinpi((hour - 6) / 12)), + Ri_SW_f=max(0.0, 450.0 * sinpi((hour - 6) / 12)), + duration=Dates.Hour(1), + ) + for hour in 1:nhours + ]) +end + +function run_maespa_example(; nhours=24, check=true) + mtg = build_maespa_scene() + mapping = maespa_mapping() + sim = run!(mtg, mapping, maespa_meteo(; nhours=nhours), check=check, executor=SequentialEx()) + return (mtg=mtg, mapping=mapping, simulation=sim) +end + +if abspath(PROGRAM_FILE) == @__FILE__ + result = run_maespa_example() + sim = result.simulation + println("leaf_count = ", length(status(sim, :Leaf))) + println("scene_transpiration = ", status(sim, :scene).scene_transpiration) + println("psi_soil = ", status(sim, :scene).psi_soil) + println("plant_A = ", only(status(sim, :plant_A, :Plant)).daily_growth) + println("plant_B = ", only(status(sim, :plant_B, :Plant)).daily_growth) +end diff --git a/examples/plantbiophysics_subsample/FvCB.jl b/examples/plantbiophysics_subsample/FvCB.jl new file mode 100644 index 000000000..cd23055a7 --- /dev/null +++ b/examples/plantbiophysics_subsample/FvCB.jl @@ -0,0 +1,182 @@ +# Generate all methods for the photosynthesis process: several meteo time-steps, components, +# over an MTG, and the mutating /non-mutating versions +@process "photosynthesis" verbose = false + +# Default policy for assimilation rates when consumed at coarser clocks. +# Mapping-level InputBindings policy still overrides this default when provided. +PlantSimEngine.output_policy(::Type{<:AbstractPhotosynthesisModel}) = (A=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()),) + + +""" +Farquhar–von Caemmerer–Berry (FvCB) model for C3 photosynthesis (Farquhar et al., 1980; +von Caemmerer and Farquhar, 1981) coupled with a conductance model. +""" +struct Fvcb{T} <: AbstractPhotosynthesisModel + Tᵣ::T + VcMaxRef::T + JMaxRef::T + RdRef::T + TPURef::T + Eₐᵣ::T + O₂::T + Eₐⱼ::T + Hdⱼ::T + Δₛⱼ::T + Eₐᵥ::T + Hdᵥ::T + Δₛᵥ::T + α::T + θ::T +end + +function Fvcb(; Tᵣ=25.0, VcMaxRef=200.0, JMaxRef=250.0, RdRef=0.6, TPURef=9999.0, Eₐᵣ=46390.0, + O₂=210.0, Eₐⱼ=29680.0, Hdⱼ=200000.0, Δₛⱼ=631.88, Eₐᵥ=58550.0, Hdᵥ=200000.0, + Δₛᵥ=629.26, α=0.425, θ=0.7) + + Fvcb(promote(Tᵣ, VcMaxRef, JMaxRef, RdRef, TPURef, Eₐᵣ, O₂, Eₐⱼ, Hdⱼ, Δₛⱼ, Eₐᵥ, Hdᵥ, Δₛᵥ, α, θ)...) +end + +function PlantSimEngine.inputs_(::Fvcb) + (aPPFD=-Inf, Tₗ=-Inf, Cₛ=-Inf) +end + +function PlantSimEngine.outputs_(::Fvcb) + (A=-Inf, Gₛ=-Inf, Cᵢ=-Inf) +end + +Base.eltype(x::Fvcb) = typeof(x).parameters[1] + +PlantSimEngine.dep(::Fvcb) = (stomatal_conductance=AbstractStomatal_ConductanceModel,) +PlantSimEngine.ObjectDependencyTrait(::Type{<:Fvcb}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:Fvcb}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.timestep_hint(::Type{<:Fvcb}) = ( + required=(Dates.Minute(1), Dates.Hour(6)), + preferred=Dates.Hour(1) +) + +PlantSimEngine.output_policy(::Type{<:Fvcb}) = ( + A=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), # from μmol m-2 s-1 to μmol m-2 timerstep-1 + Cᵢ=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()), + Gₛ=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), +) + +function arrhenius(kref, Eₐ, Tₖ, Tᵣₖ, R) + kref * exp(Eₐ * (Tₖ - Tᵣₖ) / (R * Tₖ * Tᵣₖ)) +end + +function arrhenius(kref, Eₐ, Tₖ, Tᵣₖ, Hd, Δₛ, R) + activation = arrhenius(kref, Eₐ, Tₖ, Tᵣₖ, R) + deactivation_ref = 1.0 + exp((Tᵣₖ * Δₛ - Hd) / (R * Tᵣₖ)) + deactivation = 1.0 + exp((Tₖ * Δₛ - Hd) / (R * Tₖ)) + return activation * deactivation_ref / deactivation +end + +function Γ_star(Tₖ, Tᵣₖ, R=PlantMeteo.Constants().R) + arrhenius(oftype(Tₖ, 42.75), oftype(Tₖ, 37830.0), Tₖ, Tᵣₖ, R) +end + +function get_km(Tₖ, Tᵣₖ, O₂, R=PlantMeteo.Constants().R) + KC = arrhenius(oftype(Tₖ, 404.9), oftype(Tₖ, 79430.0), Tₖ, Tᵣₖ, R) + KO = arrhenius(oftype(Tₖ, 278.4), oftype(Tₖ, 36380.0), Tₖ, Tᵣₖ, R) + return KC * (1.0 + O₂ / KO) +end + +function PlantSimEngine.run!(m::Fvcb, models, status, meteo, constants=PlantMeteo.Constants(), extra=nothing) + + # Tranform Celsius temperatures in Kelvin: + Tₖ = status.Tₗ - constants.K₀ + Tᵣₖ = m.Tᵣ - constants.K₀ + + # Temperature dependence of the parameters: + Γˢ = Γ_star(Tₖ, Tᵣₖ, constants.R) # Gamma star (CO2 compensation point) in μmol mol-1 + Km = get_km(Tₖ, Tᵣₖ, m.O₂, constants.R) # effective Michaelis–Menten coefficient for CO2 + + # Maximum electron transport rate at the given leaf temperature (μmol m-2 s-1): + JMax = arrhenius(m.JMaxRef, m.Eₐⱼ, Tₖ, Tᵣₖ, m.Hdⱼ, m.Δₛⱼ, constants.R) + # Maximum rate of Rubisco activity at the given models temperature (μmol m-2 s-1): + VcMax = arrhenius(m.VcMaxRef, m.Eₐᵥ, Tₖ, Tᵣₖ, m.Hdᵥ, m.Δₛᵥ, constants.R) + # Rate of mitochondrial respiration at the given leaf temperature (μmol m-2 s-1): + Rd = arrhenius(m.RdRef, m.Eₐᵣ, Tₖ, Tᵣₖ, constants.R) + # Rd is also described as the CO2 release in the light by processes other than the PCO + # cycle, and termed "day" respiration, or "light respiration" (Harley et al., 1986). + + # Actual electron transport rate (considering intercepted PAR and leaf temperature): + J = get_J(status.aPPFD, JMax, m.α, m.θ) # in μmol m-2 s-1 + # RuBP regeneration + Vⱼ = J / 4 + + # Stomatal conductance (mol[CO₂] m-2 s-1), dispatched on type of first argument (gs_closure): + st_closure = gs_closure(models.stomatal_conductance, models, status, meteo, extra) + + Cᵢⱼ = get_Cᵢⱼ(Vⱼ, Γˢ, status.Cₛ, Rd, models.stomatal_conductance.g0, st_closure) + + # Electron-transport-limited rate of CO2 assimilation (RuBP regeneration-limited): + Wⱼ = Vⱼ * (Cᵢⱼ - Γˢ) / (Cᵢⱼ + 2.0 * Γˢ) # also called Aⱼ + # See Von Caemmerer, Susanna. 2000. Biochemical models of leaf photosynthesis. + # Csiro publishing, eq. 2.23. + # NB: here the equation is modified because we use Vⱼ instead of J, but it is the same. + + # If Rd is larger than Wⱼ, no assimilation: + if Wⱼ - Rd < 1.0e-6 + Cᵢⱼ = Γˢ + Wⱼ = Vⱼ * (Cᵢⱼ - Γˢ) / (Cᵢⱼ + 2.0 * Γˢ) + end + + Cᵢᵥ = get_Cᵢᵥ(VcMax, Γˢ, status.Cₛ, Rd, models.stomatal_conductance.g0, st_closure, Km) + + # Rubisco-carboxylation-limited rate of CO₂ assimilation (RuBP activity-limited): + if Cᵢᵥ <= 0.0 || Cᵢᵥ > status.Cₛ + Wᵥ = 0.0 + else + Wᵥ = VcMax * (Cᵢᵥ - Γˢ) / (Cᵢᵥ + Km) + end + + # Net assimilation (μmol m-2 s-1) + status.A = min(Wᵥ, Wⱼ, 3 * m.TPURef) - Rd + + # Stomatal conductance (mol[CO₂] m-2 s-1) + PlantSimEngine.run!(models.stomatal_conductance, models, status, st_closure, extra) + + # Intercellular CO₂ concentration (Cᵢ, μmol mol) + status.Cᵢ = min(status.Cₛ, status.Cₛ - status.A / status.Gₛ) + nothing +end + +function get_J(aPPFD, JMax, α, θ) + (α * aPPFD + JMax - sqrt((α * aPPFD + JMax)^2 - 4 * α * θ * aPPFD * JMax)) / (2 * θ) +end + +function get_Cᵢⱼ(Vⱼ, Γˢ, Cₛ, Rd, g0, st_closure) + a = g0 + st_closure * (Vⱼ - Rd) + b = (1.0 - Cₛ * st_closure) * (Vⱼ - Rd) + g0 * (2.0 * Γˢ - Cₛ) - + st_closure * (Vⱼ * Γˢ + 2.0 * Γˢ * Rd) + c = -(1.0 - Cₛ * st_closure) * Γˢ * (Vⱼ + 2.0 * Rd) - + g0 * 2.0 * Γˢ * Cₛ + + return positive_root(a, b, c) +end + +function get_Cᵢᵥ(VcMAX, Γˢ, Cₛ, Rd, g0, st_closure, Km) + a = g0 + st_closure * (VcMAX - Rd) + b = (1.0 - Cₛ * st_closure) * (VcMAX - Rd) + g0 * (Km - Cₛ) - st_closure * (VcMAX * Γˢ + Km * Rd) + c = -(1.0 - Cₛ * st_closure) * (VcMAX * Γˢ + Km * Rd) - g0 * Km * Cₛ + + return positive_root(a, b, c) +end + +function max_root(a, b, c) + Δ = b^2.0 - 4.0 * a * c + x1 = (-b + sqrt(Δ)) / (2.0 * a) + x2 = (-b - sqrt(Δ)) / (2.0 * a) + return max(x1, x2) +end + +function positive_root(a, b, c) + Δ = b^2.0 - 4.0 * a * c + return Δ >= 0.0 ? (-b + sqrt(Δ)) / (2.0 * a) : 0.0 +end + +function negative_root(a, b, c) + Δ = b^2.0 - 4.0 * a * c + return Δ >= 0.0 ? (-b - sqrt(Δ)) / (2.0 * a) : 0.0 +end diff --git a/examples/plantbiophysics_subsample/Monteith.jl b/examples/plantbiophysics_subsample/Monteith.jl new file mode 100644 index 000000000..93a4367ec --- /dev/null +++ b/examples/plantbiophysics_subsample/Monteith.jl @@ -0,0 +1,737 @@ +#! Careful: this file is a copy/paste from the original model implementation in PlantBiophysics.jl (v0.16.2). It is only used for testing. +#! If you want to use this model, use the one from PlantBiophysics.jl instead, which is more up to date and maintained. + +@process "energy_balance" verbose = false +""" + black_body(T, K₀, σ) + black_body(T) + +Thermal infrared, *i.e.* longwave radiation emitted from a black body at temperature T. + +- `T`: temperature of the object in Celsius degree +- `K₀`: absolute zero (°C) +- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law) + +# Note + +`K₀` and `σ` are taken from `PlantMeteo.Constants` if not provided. + +""" +function black_body(T, K₀, σ) + Tₖ = T - K₀ + σ * (Tₖ^4.0) +end + +function black_body(T) + constants = PlantMeteo.Constants() + black_body(T, constants.K₀, constants.σ) +end + + +""" +Thermal infrared, *i.e.* longwave radiation emitted from an object at temperature T. + +- `T`: temperature of the object in Celsius degree +- `ε` object [emissivity](https://en.wikipedia.org/wiki/Emissivity) (not to confuse with ε the +ratio of molecular weights from `PlantMeteo.Constants`). A typical value for a leaf is 0.955. +- `K₀`: absolute zero (°C) +- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law) + +# Note + +`K₀` and `σ` are taken from `PlantMeteo.Constants` if not provided. + +# Examples + +```julia +# Thermal infrared radiation of water at 25 °C: +grey_body(25.0, 0.96) +``` +""" +function grey_body(T, ε, K₀, σ) + ε * black_body(T, K₀, σ) +end + +function grey_body(T, ε) + constants = PlantMeteo.Constants() + grey_body(T, ε, constants.K₀, constants.σ) +end + + +""" + net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁,K₀,σ) + net_longwave_radiation(T₁,T₂,ε₁,ε₂,F₁) + +Net longwave radiation fluxes (*i.e.* thermal radiation, W m-2) between an object and another. +The object of interest is at temperature T₁ and has an emissivity ε₁, and the object with +which it exchanges energy is at temperature T₂ and has an emissivity ε₂. + +If the result is positive, then the object of interest gain energy. + +# Arguments + +- `T₁` (Celsius degree): temperature of the target object (object 1) +- `T₂` (Celsius degree): temperature of the object with which there is potential exchange (object 2) +- `ε₁`: object 1 emissivity +- `ε₂`: object 2 emissivity +- `F₁`: view factor (0-1), *i.e.* visible fraction of object 2 from object 1 (see note) +- `K₀`: absolute zero (°C) +- `σ` (``W\\ m^{-2}\\ K^{-4}``) [Stefan-Boltzmann constant](https://en.wikipedia.org/wiki/Stefan%E2%80%93Boltzmann_law) + +# Note + +`F₁`, the view factor (also called shape factor) is a coefficient applied to the semi-hemisphere +field of view of object 1 that "sees" object 2. E.g. a leaf can be viewed as a plane. If one side +of the leaf sees only object 2 in its field of view (e.g. the sky), then `F₁ = 1`. +Then the net longwave radiation flux for this part of the leaf is multiplied by its actual +surface to get the exchange. Note that we apply reciprocity between the two objects for +the view factor (they have the same value), *i.e.*: A₁F₁₂ = A₂F₂₁. + +Then, if we take a leaf as object 1, and the sky as object 2, the visible fraction of +sky viewed by the leaf would be: + +- `0.5` if the leaf is on top of the canopy, *i.e.* the upper side of the leaf sees the sky, +the side bellow sees other leaves and the soil. +- between 0 and 0.5 if it is within the canopy and partly shaded by other objects. + +Note that `A₁` for a leaf is twice its common used leaf area, because `A₁` is the **total** +leaf area of the object that exchange energy. + +```julia +# Net thermal radiation fluxes between a leaf and the sky considering the leaf at the top of +# the canopy: +Tₗ = 25.0 ; Tₐ = 20.0 +ε₁ = 0.955 ; ε₂ = 1.0 +Ra_LW_f = net_longwave_radiation(Tₗ,Tₐ,ε₁,ε₂,1.0) +Ra_LW_f + +# Ra_LW_f is the net longwave radiation flux between the leaf and the atmosphere per surface area. +# To get the actual net longwave radiation flux we need to multiply by the surface of the +# leaf, e.g. for a leaf of 2cm²: +leaf_area = 2e-4 # in m² +Ra_LW_f * leaf_area + +# The leaf lose ~0.0055 W towards the atmosphere. +``` + +# References + +Cengel, Y, et Transfer Mass Heat. 2003. A practical approach. New York, NY, USA: McGraw-Hill. +""" +function net_longwave_radiation(T₁, T₂, ε₁, ε₂, F₁, K₀, σ) + (black_body(T₂, K₀, σ) - black_body(T₁, K₀, σ)) / (1.0 / ε₁ + 1.0 / ε₂ - 1.0) * F₁ +end + +function net_longwave_radiation(T₁, T₂, ε₁, ε₂, F₁) + constants = PlantMeteo.Constants() + net_longwave_radiation(T₁, T₂, ε₁, ε₂, F₁, constants.K₀, constants.σ) +end + +""" + gbₕ_free(Tₐ,Tₗ,d,Dₕ₀) + gbₕ_free(Tₐ,Tₗ,d) + +Leaf boundary layer conductance for heat under **free** convection (m s-1). + +# Arguments + +- `Tₐ` (°C): air temperature +- `Tₗ` (°C): leaf temperature +- `d` (m): characteristic dimension, *e.g.* leaf width (see eq. 10.9 from Monteith and Unsworth, 2013). +- `Dₕ₀ = 21.5e-6`: molecular diffusivity for heat at base temperature. Use value from +`PlantMeteo.Constants` if not provided. + +# Note + +`R` and `Dₕ₀` can be found using `PlantMeteo.Constants`. To transform in ``mol\\ m^{-2}\\ s^{-1}``, +use [`ms_to_mol`](@ref). + +# References + +Leuning, R., F. M. Kelliher, DGG de Pury, et E.-D. SCHULZE. 1995. « Leaf nitrogen, +photosynthesis, conductance and transpiration: scaling from leaves to canopies ». Plant, +Cell & Environment 18 (10): 1183‑1200. + +Monteith, John, et Mike Unsworth. 2013. Principles of environmental physics: plants, +animals, and the atmosphere. Academic Press. Paragraph 10.1.3, eq. 10.9. +""" +function gbₕ_free(Tₐ, Tₗ, d, Dₕ₀=PlantMeteo.Constants().Dₕ₀) + zeroT = zero(Tₐ) # make it type stable + + if abs(Tₗ - Tₐ) > zeroT + Gr = 1.58e8 * d^3.0 * abs(Tₗ - Tₐ) # Grashof number (Monteith and Unsworth, 2013) + # !Note: Leuning et al. (1995) use 1.6e8 (eq. E4). + # Leuning et al. (1995) eq. E3: + Gbₕ_free = 0.5 * get_Dₕ(Tₐ, Dₕ₀) * (Gr^0.25) / d + else + Gbₕ_free = zeroT + end + + return Gbₕ_free +end + + +""" + gbₕ_forced(Wind,d) + +Boundary layer conductance for heat under **forced** convection (m s-1). See eq. E1 from +Leuning et al. (1995) for more details. + +# Arguments + +- `Wind` (m s-1): wind speed +- `d` (m): characteristic dimension, *e.g.* leaf width (see eq. 10.9 from Monteith and Unsworth, 2013). + +# Notes + +`d` is the minimal dimension of the surface of an object in contact with the air. + +# References + +Leuning, R., F. M. Kelliher, DGG de Pury, et E.-D. SCHULZE. 1995. « Leaf nitrogen, +photosynthesis, conductance and transpiration: scaling from leaves to canopies ». Plant, +Cell & Environment 18 (10): 1183‑1200. +""" +function gbₕ_forced(Wind, d) + 0.003 * sqrt(Wind / d) +end + + +""" + get_Dₕ(T,Dₕ₀) + get_Dₕ(T) + +Dₕ -molecular diffusivity for heat at base temperature- from Dₕ₀ (corrected by temperature). +See Monteith and Unsworth (2013, eq. 3.10). + +# Arguments + +- `Tₐ` (°C): temperature +- `Dₕ₀`: molecular diffusivity for heat at base temperature. Use value from `PlantMeteo.Constants` +if not provided. + +# References + +Monteith, John, et Mike Unsworth. 2013. Principles of environmental physics: plants, +animals, and the atmosphere. Academic Press. Paragraph 10.1.3. +""" +function get_Dₕ(T, Dₕ₀=PlantMeteo.Constants().Dₕ₀) + Dₕ₀ * (1 + 0.007 * T) +end + +""" + ms_to_mol(G,T,P,R,K₀) + ms_to_mol(G,T,P) + +Conversion of a conductance `G` from ``m\\ s^{-1}`` to ``mol\\ m^{-2}\\ s^{-1}``. + +# Arguments + +- `G` (``m\\ s^{-1}``): conductance +- `T` (°C): air temperature +- `P` (kPa): air pressure +- `R` (``J\\ mol^{-1}\\ K^{-1}``): universal gas constant. +- `K₀` (°C): absolute zero + +# See also + +[`mol_to_ms`](@ref) for the inverse process. +""" +function ms_to_mol(G, T, P, R, K₀) + G * f_ms_to_mol(T, P, R, K₀) +end + +function ms_to_mol(G, T, P) + constants = PlantMeteo.Constants() + ms_to_mol(G, T, P, constants.R, constants.K₀) +end + +""" + ms_to_mol(G,T,P,R,K₀) + ms_to_mol(G,T,P) + +Conversion of a conductance `G` from ``mol\\ m^{-2}\\ s^{-1}`` to ``m\\ s^{-1}``. + +# Arguments + +- `G` (``m\\ s^{-1}``): conductance +- `T` (°C): air temperature +- `P` (kPa): air pressure +- `R` (``J\\ mol^{-1}\\ K^{-1}``): universal gas constant. +- `K₀` (°C): absolute zero + +# See also + +[`ms_to_mol`](@ref) for the inverse process. +""" +function mol_to_ms(G, T, P, R, K₀) + G / f_ms_to_mol(T, P, R, K₀) +end + +function mol_to_ms(G, T, P) + constants = PlantMeteo.Constants() + mol_to_ms(G, T, P, constants.R, constants.K₀) +end + +""" +Conversion factor between conductance in ``m\\ s^{-1}`` to ``mol\\ m^{-2}\\ s^{-1}``. + +# Arguments + +- `T` (°C): air temperature +- `P` (kPa): air pressure +- `R` (``J\\ mol^{-1}\\ K^{-1}``): universal gas constant. +- `K₀` (°C): absolute zero +""" +function f_ms_to_mol(T, P, R, K₀) + (P * 1000) / (R * (T - K₀)) +end + +""" + gbh_to_gbw(gbh, Gbₕ_to_Gbₕ₂ₒ = PlantMeteo.Constants().Gbₕ_to_Gbₕ₂ₒ) + gbw_to_gbh(gbh, Gbₕ_to_Gbₕ₂ₒ = PlantMeteo.Constants().Gbₕ_to_Gbₕ₂ₒ) + +Boundary layer conductance for water vapor from boundary layer conductance for heat. + +# Arguments + +- `gbh` (m s-1): boundary layer conductance for heat under mixed convection. +- `Gbₕ_to_Gbₕ₂ₒ`: conversion factor. + +# Note + +Gbₕ is the sum of free and forced convection. See [`gbₕ_free`](@ref) and [`gbₕ_forced`](@ref). +""" +function gbh_to_gbw(gbh, Gbₕ_to_Gbₕ₂ₒ=PlantMeteo.Constants().Gbₕ_to_Gbₕ₂ₒ) + gbh * Gbₕ_to_Gbₕ₂ₒ +end + +function gbw_to_gbh(gbh, Gbₕ_to_Gbₕ₂ₒ=PlantMeteo.Constants().Gbₕ_to_Gbₕ₂ₒ) + gbh / Gbₕ_to_Gbₕ₂ₒ +end + + +""" + gsc_to_gsw(Gₛ, Gsc_to_Gsw = PlantMeteo.Constants().Gsc_to_Gsw) + +Conversion of a stomatal conductance for CO₂ into stomatal conductance for H₂O. +""" +function gsc_to_gsw(Gₛ, Gsc_to_Gsw=PlantMeteo.Constants().Gsc_to_Gsw) + Gₛ * Gsc_to_Gsw +end + +""" + gsw_to_gsc(Gₛ, Gsc_to_Gsw = PlantMeteo.Constants().Gsc_to_Gsw) + +Conversion of a stomatal conductance for H₂O into stomatal conductance for CO₂. +""" +function gsw_to_gsc(Gₛ, Gsc_to_Gsw=PlantMeteo.Constants().Gsc_to_Gsw) + Gₛ / Gsc_to_Gsw +end + +""" +γ_star(γ, a_sh, a_s, rbv, Rsᵥ, Rbₕ) + +γ∗, the apparent value of psychrometer constant (kPa K−1). + +# Arguments + +- `γ` (kPa K−1): psychrometer constant +- `aₛₕ` (1,2): number of faces exchanging heat fluxes (see Schymanski et al., 2017) +- `aₛᵥ` (1,2): number of faces exchanging water fluxes (see Schymanski et al., 2017) +- `Rbᵥ` (s m-1): boundary layer resistance to water vapor +- `Rsᵥ` (s m-1): stomatal resistance to water vapor +- `Rbₕ` (s m-1): boundary layer resistance to heat + +# Note + +Using the corrigendum from Schymanski et al. (2017) in here so the definition of +[`latent_heat`](@ref) remains generic. + +Not to be confused with [`Γ_star`](@ref) the CO₂ compensation point. + +# References + +Monteith, John L., et Mike H. Unsworth. 2013. « Chapter 13 - Steady-State Heat Balance: (i) +Water Surfaces, Soil, and Vegetation ». In Principles of Environmental Physics (Fourth Edition), +edited by John L. Monteith et Mike H. Unsworth, 217‑47. Boston: Academic Press. + +Schymanski, Stanislaus J., et Dani Or. 2017. Leaf-Scale Experiments Reveal an Important +Omission in the Penman–Monteith Equation ». Hydrology and Earth System Sciences 21 (2): 685‑706. +https://doi.org/10.5194/hess-21-685-2017. +""" +function γ_star(γ, aₛₕ, aₛᵥ, Rbᵥ, Rsᵥ, Rbₕ) + γ * aₛₕ / aₛᵥ * (Rbᵥ + Rsᵥ) / Rbₕ # rv + Rsᵥ= Boundary + stomatal conductance to water vapour +end + +""" + λE_to_E(λE, λ, Mₕ₂ₒ=PlantMeteo.Constants().Mₕ₂ₒ) + E_to_λE(E, λ, Mₕ₂ₒ=PlantMeteo.Constants().Mₕ₂ₒ) + +Conversion from latent heat (W m-2) to evaporation (mol[H₂O] m-2 s-1) or the +opposite (`E_to_λE`). + +# Arguments + +- `λE`: latent heat flux (W m-2) +- `E`: water evaporation (mol[H₂O] m-2 s-1) +- `λ` (J kg-1): latent heat of vaporization +- `Mₕ₂ₒ = 18.0e-3` (kg mol-1): Molar mass for water. + +# Note + +`λ` can be computed using: + + λ = latent_heat_vaporization(T, constants.λ₀) + +It is also directly available from the [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere) structure, and by extention in [`Weather`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Weather). + +To convert E from mol[H₂O] m-2 s-1 to mm s-1 you can simply do: + + E_mms = E_mol / constants.Mₕ₂ₒ + +mm[H₂O] s-1 is equivalent to kg[H₂O] m-2 s-1, wich is equivalent to l[H₂O] m-2 s-1. + +""" +function λE_to_E(λE, λ, Mₕ₂ₒ=PlantMeteo.Constants().Mₕ₂ₒ) + λE / λ * Mₕ₂ₒ +end + +function E_to_λE(E, λ, Mₕ₂ₒ=PlantMeteo.Constants().Mₕ₂ₒ) + E / Mₕ₂ₒ * λ +end + +""" +Struct to hold parameter and values for the energy model close to the one in +Monteith and Unsworth (2013) + +# Arguments + +- `aₛₕ = 2`: number of faces of the object that exchange sensible heat fluxes +- `aₛᵥ = 1`: number of faces of the object that exchange latent heat fluxes (hypostomatous => 1) +- `ε = 0.955`: emissivity of the object +- `maxiter = 10`: maximal number of iterations allowed to close the energy balance +- `ΔT = 0.01` (°C): maximum difference in object temperature between two iterations to consider convergence + +# Examples + +```julia +energy_model = Monteith() # a leaf in an illuminated chamber +``` +""" +struct Monteith{T,S} <: AbstractEnergy_BalanceModel + aₛₕ::S + aₛᵥ::S + ε::T + maxiter::S + ΔT::T +end + +function Monteith(; aₛₕ=2, aₛᵥ=1, ε=0.955, maxiter=10, ΔT=0.01) + param_int = promote(aₛₕ, aₛᵥ, maxiter) + param_float = promote(ε, ΔT) + Monteith(param_int[1], param_int[2], param_float[1], param_int[3], param_float[2]) +end + +function PlantSimEngine.inputs_(::Monteith) + (Ra_SW_f=-Inf, sky_fraction=-Inf, d=-Inf) +end + +function PlantSimEngine.outputs_(::Monteith) + ( + Tₗ=-Inf, Rn=-Inf, Ra_LW_f=-Inf, H=-Inf, λE=-Inf, Cₛ=-Inf, Cᵢ=-Inf, + A=-Inf, Gₛ=-Inf, Gbₕ=-Inf, Dₗ=-Inf, Gbc=-Inf, iter=typemin(Int) + ) +end + +Base.eltype(x::Monteith) = typeof(x).parameters[1] +PlantSimEngine.ObjectDependencyTrait(::Type{<:Monteith}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:Monteith}) = PlantSimEngine.IsTimeStepIndependent() +# Multi-rate default for energy balance: keep relatively fine cadence. +PlantSimEngine.timestep_hint(::Type{<:Monteith}) = ( + required=(Dates.Minute(1), Dates.Hour(2)), + preferred=Dates.Hour(1) +) +PlantSimEngine.output_policy(::Type{<:Monteith}) = ( + A=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), + Tₗ=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()), + Rn=PlantSimEngine.Integrate(PlantMeteo.RadiationEnergy()), # W m-2 to MJ m-2 timestep-1 + Ra_LW_f=PlantSimEngine.Integrate(PlantMeteo.RadiationEnergy()), + H=PlantSimEngine.Integrate(PlantMeteo.RadiationEnergy()), + λE=PlantSimEngine.Integrate(PlantMeteo.RadiationEnergy()), + Cₛ=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()), + Cᵢ=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()), + Gₛ=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), + Gbₕ=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), + Dₗ=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()), + Gbc=PlantSimEngine.Integrate(PlantMeteo.DurationSumReducer()), + iter=PlantSimEngine.Integrate(PlantMeteo.MeanReducer()) +) + +PlantSimEngine.dep(::Monteith) = (photosynthesis=AbstractPhotosynthesisModel,) + +""" + run!(::Monteith, models, status, meteo, constants=Constants()) + +Leaf energy balance according to Monteith and Unsworth (2013), and corrigendum from +Schymanski et al. (2017). The computation is close to the one from the MAESPA model (Duursma +et al., 2012, Vezy et al., 2018) here. The leaf temperature is computed iteratively to close +the energy balance using the mass flux (~ Rn - λE). + +# Arguments + +- `::Monteith`: a Monteith model, usually from a model list (*i.e.* m.energy_balance) +- `models`: A `ModelMapping` struct holding the parameters for the model with +initialisations for: + - `Ra_SW_f` (W m-2): net shortwave radiation (PAR + NIR). Often computed from a light interception model + - `sky_fraction` (0-2): view factor between the object and the sky for both faces (see details). + - `d` (m): characteristic dimension, *e.g.* leaf width (see eq. 10.9 from Monteith and Unsworth, 2013). +- `status`: the status of the model, usually the model list status (*i.e.* leaf.status) +- `meteo`: meteorology structure, see [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere) +- `constants = PlantMeteo.Constants()`: physical constants. See `PlantMeteo.Constants` for more details + +# Details + +The sky_fraction in the variables is equal to 2 if all the leaf is viewing is sky (e.g. in a +controlled chamber), 1 if the leaf is *e.g.* up on the canopy where the upper side of the +leaf sees the sky, and the side bellow sees soil + other leaves that are all considered at +the same temperature than the leaf, or less than 1 if it is partly shaded. + +# Notes + +If you want the algorithm to print a message whenever it does not reach convergence, use the +debugging mode by executing this in the REPL: `ENV["JULIA_DEBUG"] = PlantBiophysics`. + +More information [here](https://docs.julialang.org/en/v1/stdlib/Logging/#Environment-variables). + +# Examples + +```julia +meteo = Atmosphere(T = 22.0, Wind = 0.8333, P = 101.325, Rh = 0.4490995) + +# Using a constant value for Gs: +leaf = ModelMapping( + energy_balance = Monteith(), + photosynthesis = Fvcb(), + stomatal_conductance = ConstantGs(0.0, 0.0011), + status = (Ra_SW_f = 13.747, sky_fraction = 1.0, d = 0.03) +) + +run!(leaf,meteo) +leaf.status.Rn +julia> 12.902547446281233 + +# Using the model from Medlyn et al. (2011) for Gs: +leaf = ModelMapping( + energy_balance = Monteith(), + photosynthesis = Fvcb(), + stomatal_conductance = Medlyn(0.03, 12.0), + status = (Ra_SW_f = 13.747, sky_fraction = 1.0, aPPFD = 1500.0, d = 0.03) +) + +out_sim = run!(leaf,meteo) +out_sim[:Rn] +out_sim[:Ra_LW_f] +out_sim[:A] + +df = PlantSimEngine.convert_outputs(out_sim, DataFrame) +``` + +# References + +Duursma, R. A., et B. E. Medlyn. 2012. « MAESPA: a model to study interactions between water +limitation, environmental drivers and vegetation function at tree and stand levels, with an +example application to [CO2] × drought interactions ». Geoscientific Model Development 5 (4): +919‑40. https://doi.org/10.5194/gmd-5-919-2012. + +Monteith, John L., et Mike H. Unsworth. 2013. « Chapter 13 - Steady-State Heat Balance: (i) +Water Surfaces, Soil, and Vegetation ». In Principles of Environmental Physics (Fourth Edition), +edited by John L. Monteith et Mike H. Unsworth, 217‑47. Boston: Academic Press. + +Schymanski, Stanislaus J., et Dani Or. 2017. « Leaf-Scale Experiments Reveal an Important +Omission in the Penman–Monteith Equation ». Hydrology and Earth System Sciences 21 (2): 685‑706. +https://doi.org/10.5194/hess-21-685-2017. + +Vezy, Rémi, Mathias Christina, Olivier Roupsard, Yann Nouvellon, Remko Duursma, Belinda Medlyn, +Maxime Soma, et al. 2018. « Measuring and modelling energy partitioning in canopies of varying +complexity using MAESPA model ». Agricultural and Forest Meteorology 253‑254 (printemps): 203‑17. +https://doi.org/10.1016/j.agrformet.2018.02.005. +""" +function PlantSimEngine.run!(::Monteith, models, status, meteo, constants=PlantMeteo.Constants(), extra=nothing) + + # Initialisations + status.Tₗ = meteo.T - 0.2 + Tₗ_new = zero(meteo.T) + status.Cₛ = meteo.Cₐ + status.Dₗ = PlantMeteo.e_sat(status.Tₗ) - PlantMeteo.e_sat(meteo.T) * meteo.Rh + γˢ = Rbₕ = Δ = zero(meteo.T) + status.Rn = status.Ra_SW_f + iter = 0 + # ?NB: We use iter = 0 and not 1 to get the right number of iterations at the end + # of the for loop, because we use iter += 1 at the end (so it increments once again) + + # Iterative resolution of the energy balance + for i in 1:models.energy_balance.maxiter + + # Update A, Gₛ, Cᵢ from models.status: + PlantSimEngine.run!(models.photosynthesis, models, status, meteo, constants, extra) + + # Stomatal resistance to water vapor + Rsᵥ = 1.0 / (gsc_to_gsw(mol_to_ms(status.Gₛ, meteo.T, meteo.P, constants.R, constants.K₀), + constants.Gsc_to_Gsw)) + + # Re-computing the net radiation according to simulated leaf temperature: + status.Ra_LW_f = net_longwave_radiation(status.Tₗ, meteo.T, models.energy_balance.ε, meteo.ε, + status.sky_fraction, constants.K₀, constants.σ) + #= ? NB: we use the sky fraction here (0-2) instead of the view factor (0-1) because: + - we consider both sides of the leaf at the same time (1 -> leaf sees sky on one face) + - we consider all objects in the scene have the same temperature as the leaf + of interest except the atmosphere. So the leaf exchange thermal energy_balance only with + the atmosphere. =# + # status.Ra_LW_f = (grey_body(meteo.T,1.0) - grey_body(status.Tₗ, 1.0))*status.sky_fraction + + status.Rn = status.Ra_SW_f + status.Ra_LW_f + + # Leaf boundary conductance for heat (m s-1), one sided: + status.Gbₕ = gbₕ_free(meteo.T, status.Tₗ, status.d, constants.Dₕ₀) + + gbₕ_forced(meteo.Wind, status.d) + # NB, in MAESPA we use Rni so we add the radiation conductance also (not here) + + # Leaf boundary resistance for heat (s m-1): + Rbₕ = 1 / status.Gbₕ + + # Leaf boundary resistance for water vapor (s m-1): + Rbᵥ = 1 / gbh_to_gbw(status.Gbₕ) + + # Leaf boundary conductance for CO₂ (mol[CO₂] m-2 s-1): + status.Gbc = ms_to_mol(status.Gbₕ, meteo.T, meteo.P, constants.R, constants.K₀) / + constants.Gbc_to_Gbₕ + + # Update Cₛ using boundary layer conductance to CO₂ and assimilation: + status.Cₛ = min(meteo.Cₐ, meteo.Cₐ - status.A / (status.Gbc * models.energy_balance.aₛᵥ)) + + # Apparent value of psychrometer constant (kPa K−1) + γˢ = γ_star(meteo.γ, models.energy_balance.aₛₕ, models.energy_balance.aₛᵥ, Rbᵥ, Rsᵥ, Rbₕ) + + status.λE = latent_heat(status.Rn, meteo.VPD, γˢ, Rbₕ, meteo.Δ, meteo.ρ, + models.energy_balance.aₛₕ, constants.Cₚ) + + # If potential evaporation is needed, here is how to compute it: + # γˢₑ = γ_star(meteo.γ, energy_balance.aₛₕ, 1, Rbᵥ, 1.0e-9, Rbₕ) # Rsᵥ is inf. low + # Ev = latent_heat(status.Rn, meteo.VPD, γˢₑ, Rbₕ, meteo.Δ, meteo.ρ, energy_balance.aₛₕ, constants.Cₚ) + + Tₗ_new = meteo.T + (status.Rn - status.λE) / + (meteo.ρ * constants.Cₚ * (models.energy_balance.aₛₕ / Rbₕ)) + + if abs(Tₗ_new - status.Tₗ) <= models.energy_balance.ΔT + break + end + + status.Tₗ = Tₗ_new + + # Vapour pressure difference between the surface and the saturation vapour pressure: + status.Dₗ = PlantMeteo.e_sat(status.Tₗ) - PlantMeteo.e_sat(meteo.T) * meteo.Rh + + iter += 1 + end + + status.H = sensible_heat(status.Rn, meteo.VPD, γˢ, Rbₕ, meteo.Δ, meteo.ρ, + models.energy_balance.aₛₕ, constants.Cₚ) + + status.iter = iter + + @debug begin + if iter == models.energy_balance.maxiter + "`run!` algorithm did not converge. Please check the value." + end + end + + # Transpiration (mol[H₂O] m-2 s-1): + # ET = status.λE / meteo.λ * constants.Mₕ₂ₒ + # ET / constants.Mₕ₂ₒ to get mm s-1 <=> kg m-2 s-1 <=> l m-2 s-1 + + nothing +end + +""" + latent_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, Cₚ) + latent_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ) + +λE -the latent heat flux (W m-2)- using the Monteith and Unsworth (2013) definition corrected by +Schymanski et al. (2017), eq.22. + +- `Rn` (W m-2): net radiation. Carefull: not the isothermal net radiation +- `VPD` (kPa): air vapor pressure deficit +- `γˢ` (kPa K−1): apparent value of psychrometer constant (see `PlantMeteo.γ_star`) +- `Rbₕ` (s m-1): resistance for heat transfer by convection, i.e. resistance to sensible heat +- `Δ` (KPa K-1): rate of change of saturation vapor pressure with temperature (see `PlantMeteo.e_sat_slope`) +- `ρ` (kg m-3): air density of moist air. +- `aₛₕ` (1,2): number of sides that exchange energy for heat (2 for leaves) +- `Cₚ` (J K-1 kg-1): specific heat of air for constant pressure + +# References + +Monteith, J. and Unsworth, M., 2013. Principles of environmental physics: plants, animals, and the atmosphere. Academic Press. See eq. 13.33. + +Schymanski et al. (2017), Leaf-scale experiments reveal an important omission in the Penman–Monteith equation, +Hydrology and Earth System Sciences. DOI: https://doi.org/10.5194/hess-21-685-2017. See equ. 22. + +# Examples + +```julia +Tₐ = 20.0 ; P = 100.0 ; +ρ = air_density(Tₐ, P) # in kg m-3 +Δ = e_sat_slope(Tₐ) + +latent_heat(300.0, 2.0, 0.1461683, 50.0, Δ, ρ, 2.0) +``` +""" +function latent_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, Cₚ) + (Δ * Rn + ρ * Cₚ * VPD * (aₛₕ / Rbₕ)) / (Δ + γˢ) +end + +function latent_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ) + latent_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, PlantMeteo.Constants().Cₚ) +end + + +""" + sensible_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, Cₚ) + sensible_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ) + +H -the sensible heat flux (W m-2)- using the Monteith and Unsworth (2013) definition corrected by +Schymanski et al. (2017), eq.22. + +- `Rn` (W m-2): net radiation. Carefull: not the isothermal net radiation +- `VPD` (kPa): air vapor pressure deficit +- `γˢ` (kPa K−1): apparent value of psychrometer constant (see `PlantMeteo.γ_star`) +- `Rbₕ` (s m-1): resistance for heat transfer by convection, i.e. resistance to sensible heat +- `Δ` (KPa K-1): rate of change of saturation vapor pressure with temperature (see `PlantMeteo.e_sat_slope`) +- `ρ` (kg m-3): air density of moist air. +- `aₛₕ` (1,2): number of sides that exchange energy for heat (2 for leaves) +- `Cₚ` (J K-1 kg-1): specific heat of air for constant pressure + +# References + +Monteith, J. and Unsworth, M., 2013. Principles of environmental physics: plants, animals, and the atmosphere. Academic Press. See eq. 13.33. + +Schymanski et al. (2017), Leaf-scale experiments reveal an important omission in the Penman–Monteith equation, +Hydrology and Earth System Sciences. DOI: https://doi.org/10.5194/hess-21-685-2017. See equ. 22. + +# Examples + +```julia +Tₐ = 20.0 ; P = 100.0 ; +ρ = air_density(Tₐ, P) # in kg m-3 +Δ = PlantMeteo.e_sat_slope(Tₐ) + +sensible_heat(300.0, 2.0, 0.1461683, 50.0, Δ, ρ, 2.0) +``` +""" +function sensible_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, Cₚ) + (γˢ * Rn - ρ * Cₚ * VPD * (aₛₕ / Rbₕ)) / (Δ + γˢ) +end + +function sensible_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ) + sensible_heat(Rn, VPD, γˢ, Rbₕ, Δ, ρ, aₛₕ, PlantMeteo.Constants().Cₚ) +end diff --git a/examples/plantbiophysics_subsample/Tuzet.jl b/examples/plantbiophysics_subsample/Tuzet.jl new file mode 100644 index 000000000..c677fd34f --- /dev/null +++ b/examples/plantbiophysics_subsample/Tuzet.jl @@ -0,0 +1,128 @@ +#! Careful: this file is more or less a copy/paste from the original model implementation in PlantBiophysics.jl (v0.16.2). It is only used for testing. +#! If you want to use this model, use the one from PlantBiophysics.jl instead, which is more up to date and maintained. + +# Generate all methods for the stomatal conductance process: several meteo time-steps, components, +# over an MTG, and the mutating /non-mutating versions +@process "stomatal_conductance" verbose = false + +# Default policy for stomatal conductance when consumed at coarser clocks. +# Conductance is typically summarized over a window rather than accumulated. +PlantSimEngine.output_policy(::Type{<:AbstractStomatal_ConductanceModel}) = (Gₛ=PlantSimEngine.Aggregate(PlantMeteo.DurationSumReducer()),) + +# Gs is used a little bit differently compared to the other processes. We use two forms: +# the stomatal closure and the full computation of Gs +function PlantSimEngine.run!(Gs::Gsm, models, status, gs_closure, extra) where {Gsm<:AbstractStomatal_ConductanceModel} + status.Gₛ = max( + models.stomatal_conductance.gs_min, + models.stomatal_conductance.g0 + gs_closure * status.A + ) +end + +function PlantSimEngine.run!(Gs::Gsm, models, status, meteo, constants, extra) where {Gsm<:AbstractStomatal_ConductanceModel} + status.Gₛ = max( + models.stomatal_conductance.gs_min, + models.stomatal_conductance.g0 + gs_closure(models.stomatal_conductance, models, status, meteo, constants, extra) * status.A + ) +end + +""" +Tuzet et al. (2003) stomatal conductance model for CO₂. + +# Arguments + +- `g0`: intercept (μmol m⁻² s⁻¹). +- `g1`: slope. +- `Ψᵥ`: leaf water potential at which stomatal conductance is halved (MPa). +- `sf`: sensitivity factor for stomatal closure. +- `Γ`: CO₂ compensation point (mol mol⁻¹). +- `gs_min`: residual conductance (μmol m⁻² s⁻¹). + +# Variables + +- `Ψₗ`: leaf water potential (MPa). +- `Cₛ`: CO₂ concentration at the leaf surface (μmol mol⁻¹). +- `A`: CO₂ assimilation rate (μmol m⁻² s⁻¹). +- `Gₛ`: stomatal conductance (μmol m⁻² s⁻¹). + +# Note + +The CO₂ compensation point represents the concentration of CO₂ at which photosynthesis and respiration are balanced, +and it is typically a small positive value around 30–50 μmol mol⁻¹ under normal atmospheric conditions. + +This implementation uses Cₛ instead of Cᵢ. + +# Examples + +```julia +using PlantMeteo, PlantSimEngine, PlantBiophysics +meteo = Atmosphere(T = 20.0, Wind = 1.0, P = 101.3, Rh = 0.65) + +leaf = + ModelMapping( + stomatal_conductance = Tuzet(0.03, 12.0, -1.5, 2.0, 30.0), + status = (Cₛ = 380.0, Ψₗ = -1.0) + ) +run!(leaf, meteo) +``` + +# References + +Tuzet, A., Perrier, A., & Leuning, R. (2003). A coupled model of stomatal conductance, photosynthesis and transpiration. Plant, Cell & Environment, 26(7), 1097-1116. +""" +struct Tuzet{T} <: AbstractStomatal_ConductanceModel + g0::T + g1::T + Ψᵥ::T + sf::T + Γ::T + gs_min::T +end + +Tuzet(g0, g1, Ψᵥ, sf, Γ, gs_min=oftype(g0, 0.001)) = Tuzet(promote(g0, g1, Ψᵥ, sf, Γ, gs_min)) +Tuzet(; g0, g1, Ψᵥ, sf, Γ, gs_min=0.001) = Tuzet(g0, g1, Ψᵥ, sf, Γ, gs_min) + +function PlantSimEngine.inputs_(::Tuzet) + (Ψₗ=-Inf, Cₛ=-Inf) +end + +function PlantSimEngine.outputs_(::Tuzet) + (Gₛ=-Inf,) +end + +Base.eltype(::Tuzet{T}) where T = T + +""" + gs_closure(::Tuzet, models, status, meteo, constants=nothing, extra=nothing) + +Stomatal closure for CO₂ according to Tuzet et al. (2003). + +# Arguments + +- `::Tuzet`: an instance of the `Tuzet` model type. +- `models::ModelMapping`: A `ModelMapping` struct holding the parameters for the models. +- `status`: A status struct holding the variables for the models. +- `meteo`: meteorology structure, see [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere). Is not used in this model. +- `constants`: A constants struct holding the constants for the models. Is not used in this model. +- `extra`: A struct holding the extra variables for the models. Is not used in this model. + +# Details + +The stomatal conductance is calculated as: + + FPSIF = (1 + exp(sf * psiv)) / (1 + exp(sf * (psiv - Ψₗ))) + GSDIVA = g0 + (g1 / (Cₛ - Γ)) * FPSIF + +where `Γ` is the CO₂ compensation point. +""" +function gs_closure(m::Tuzet, models, status, meteo, constants=nothing, extra=nothing) + fpsif = (1 + exp(m.sf * m.Ψᵥ)) / + (1 + exp(m.sf * (m.Ψᵥ - status.Ψₗ))) + (m.g1 / (status.Cₛ - m.Γ)) * fpsif +end + +PlantSimEngine.ObjectDependencyTrait(::Type{<:Tuzet}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:Tuzet}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.timestep_hint(::Type{<:Tuzet}) = ( + required=(Dates.Minute(1), Dates.Hour(6)), + preferred=Dates.Hour(1) +) diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 1151b8f7a..b13fdf334 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -12,6 +12,7 @@ import CSV # For reading csv files with variables() import AbstractTrees import Term import Markdown +import Base: position # For multi-threading: import FLoops: @floop, @init, ThreadedEx, SequentialEx, DistributedEx @@ -36,6 +37,7 @@ include("doc_templates/mtg-related.jl") # Models: include("Abstract_model_structs.jl") +include("mtg/node_mapping_types.jl") # Multi-rate scaffolding: include("time/multirate.jl") @@ -44,14 +46,17 @@ include("time/multirate.jl") include("component_models/Status.jl") include("component_models/RefVector.jl") +# Unified scene/object API: +include("scene_object_api.jl") + # Simulation table (time-step table, from PlantMeteo): include("component_models/TimeStepTable.jl") # Declaring the dependency graph include("dependencies/dependency_graph.jl") -# List of models: -include("component_models/ModelList.jl") # deprecated, to be removed in favor of ModelMapping +# Single-scale model container used internally by ModelMapping: +include("component_models/SingleScaleModelSet.jl") include("mtg/MultiScaleModel.jl") include("mtg/ModelSpec.jl") include("mtg/mapping/mapping.jl") @@ -68,6 +73,7 @@ include("dependencies/hard_dependencies.jl") include("dependencies/traversal.jl") include("dependencies/is_graph_cyclic.jl") include("dependencies/printing.jl") +include("dependencies/update_dependencies.jl") include("dependencies/dependencies.jl") include("dependencies/get_model_in_dependency_graph.jl") @@ -103,6 +109,10 @@ include("time/runtime/input_resolution.jl") include("time/runtime/publishers.jl") include("time/runtime/output_export.jl") include("time/runtime/meteo_sampling.jl") +include("time/runtime/environment_backends.jl") + +# Domain-aware simulation scaffolding: +include("domains/domain_simulation.jl") # Simulation: include("run.jl") @@ -125,17 +135,44 @@ export OutputCache, HoldLastCache, InterpolateCache, IntegrateCache, AggregateCa export TemporalState export OutputRequest, collect_outputs export effective_rate_summary -export ModelList, MultiScaleModel, ModelMapping, ModelSpec, TimeStepModel, InputBindings, MeteoBindings, MeteoWindow, OutputRouting, ScopeModel +export Scene, Object, ObjectId, SceneRegistry, ObjectTemplate, ObjectInstance, Override +export register_object!, remove_object!, reparent_object!, move_object!, update_geometry!, refresh_bindings! +export bindings_dirty, environment_bindings_dirty, scene_revision, environment_revision +export compiled_bindings, compiled_environment_bindings, mark_environment_binding_dirty! +export refresh_environment_bindings!, compile_environment_bindings, bind_environment +export object_ids, scene_objects, resolve_object_ids, resolve_objects, explain_objects, explain_instances, explain_scopes +export geometry, position, bounds +export CompiledScene, CompiledSceneApplication, CompiledSceneInputBinding, CompiledSceneCallBinding +export compile_scene, explain_scene_applications, explain_bindings, explain_calls, explain_writers +export ObjectRefVector, input_carrier, input_value, has_reference_carrier +export SceneRunContext, SceneCallTarget +export CompiledEnvironmentBinding, CompiledEnvironmentBindings, explain_environment_bindings +export SceneScope, Self, SelfPlant, Ancestor, Scope, Kind, Species, Scale, Relation +export One, OptionalOne, Many, ObjectAddress, object_address +export Input, Call, AppliesTo, Inputs, Calls, TimeStep, Environment +export application_name, applies_to, value_inputs, model_calls, environment_config +export MultiScaleModel, ModelMapping, ModelSpec, SameScale, Updates, TimeStepModel, InputBindings, MeteoBindings, MeteoWindow, OutputRouting, ScopeModel export resolved_model_specs, explain_model_specs +export Domain, SimulationMapping, DomainSimulation, DomainModelKey, AllDomains, HardDomains +export Route, DomainRouteTarget, RouteCardinality +export ManyToOneVector, ManyToOneAggregate, OneToManyBroadcast, SpatialSample, SpatialScatterAdd +export dependency_values, dependency_targets, dependency_target, model_target, run_target!, run_call!, ModelTarget, explain_domains, explain_domain_models, explain_domain_statuses, explain_schedule, explain_domain_dependencies, explain_routes export RMSE, NRMSE, EF, dr export Status, TimeStepTable, status export init_status! -export add_organ! +export add_organ!, remove_organ!, reparent_organ! export @process, process export to_initialize, is_initialized, init_variables, dep export inputs, outputs, variables, convert_outputs export timespec, output_policy, timestep_hint, meteo_hint -export input_bindings, meteo_bindings, meteo_window, output_routing, model_scope +export input_bindings, meteo_bindings, meteo_window, output_routing, model_scope, updates +export meteo_inputs, meteo_inputs_, meteo_outputs, meteo_outputs_ +export validate_meteo_inputs +export AbstractEnvironmentBackend, EnvironmentSupport, GlobalConstant +export environment_backend, environment_variables, base_step_seconds +export sample, sample_environment, scatter!, update_index! +export scatter_environment_outputs! +export explain_environment export run! export fit diff --git a/src/checks/dimensions.jl b/src/checks/dimensions.jl index 6cc929113..605e5ca1c 100644 --- a/src/checks/dimensions.jl +++ b/src/checks/dimensions.jl @@ -43,7 +43,7 @@ ERROR: DimensionMismatch: Component status has a vector variable : var1 implying check_dimensions(component, weather) = check_dimensions(DataFormat(weather), component, weather) # Here we add methods for applying to a component, an array or a dict of: -function check_dimensions(component::T, w) where {T<:ModelList} +function check_dimensions(component::SingleScaleModelSet, w) check_dimensions(status(component), w) end @@ -52,14 +52,14 @@ function check_dimensions(component::ModelMapping{SingleScale}, w) end # for several components as an array -function check_dimensions(component::T, weather) where {T<:AbstractArray{<:ModelList}} +function check_dimensions(component::AbstractVector{<:SingleScaleModelSet}, weather) for i in component check_dimensions(i, weather) end end # for several components as a Dict -function check_dimensions(component::T, weather) where {T<:AbstractDict{N,<:ModelList} where {N}} +function check_dimensions(component::AbstractDict{<:Any,<:SingleScaleModelSet}, weather) for (key, val) in component check_dimensions(val, weather) end diff --git a/src/component_models/ModelList.jl b/src/component_models/SingleScaleModelSet.jl similarity index 83% rename from src/component_models/ModelList.jl rename to src/component_models/SingleScaleModelSet.jl index 3c97efbe3..7f400b65b 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/SingleScaleModelSet.jl @@ -1,7 +1,7 @@ """ - ModelList(models::M, status::S) - ModelList(; + SingleScaleModelSet(models::M, status::S) + SingleScaleModelSet(; status=nothing, type_promotion=nothing, variables_check=true, @@ -36,7 +36,7 @@ the `status` field in the input, you'll have to implement a method for `add_mode adds the models variables to the type in case it is not fully initialized. The default method is compatible with any type that implements the `Tables.jl` interface (*e.g.* DataFrame), and `NamedTuples`. -Note that `ModelList`makes a copy of the input `status` if it does not list all needed variables. +Note that `SingleScaleModelSet`makes a copy of the input `status` if it does not list all needed variables. ## Examples @@ -44,53 +44,54 @@ We'll use the dummy models from the `dummy.jl` in the examples folder of the pac implements three dummy processes: `Process1Model`, `Process2Model` and `Process3Model`, with one model implementation each: `Process1Model`, `Process2Model` and `Process3Model`. -```jldoctest 1 +```julia julia> using PlantSimEngine; ``` Including example processes and models: -```jldoctest 1 +```julia julia> using PlantSimEngine.Examples; ``` -```jldoctest 1 -julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model()); +```julia +julia> models = SingleScaleModelSet(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model()); [ Info: Some variables must be initialized before simulation: (process1 = (:var1, :var2), process2 = (:var1,)) (see `to_initialize()`) ``` -```jldoctest 1 +```julia julia> typeof(models) -ModelList{@NamedTuple{process1::Process1Model, process2::Process2Model, process3::Process3Model}, Status{(:var5, :var4, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}} +SingleScaleModelSet{@NamedTuple{process1::Process1Model, process2::Process2Model, process3::Process3Model}, Status{(:var5, :var4, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}} ``` -No variables were given as keyword arguments, that means that the status of the ModelList is not +No variables were given as keyword arguments, that means that the status of the SingleScaleModelSet is not set yet, and all variables are initialized to their default values given in the inputs and outputs (usually `typemin(Type)`, *i.e.* `-Inf` for floating point numbers). This component cannot be simulated yet. To know which variables we need to initialize for a simulation, we use [`to_initialize`](@ref): -```jldoctest 1 +```julia julia> to_initialize(models) (process1 = (:var1, :var2), process2 = (:var1,)) ``` -We can now provide values for these variables in the `status` field, and simulate the `ModelList`, -*e.g.* for `process3` (coupled with `process1` and `process2`): +We can now provide values for these variables in the `status` field. Direct +`run!(::SingleScaleModelSet, ...)` has been removed; wrap the models in a `ModelMapping` +before running: -```jldoctest 1 -julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0, var2=0.3)); +```julia +julia> mapping = ModelMapping(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0, var2=0.3)); ``` -```jldoctest 1 +```julia julia> meteo = Atmosphere(T = 22.0, Wind = 0.8333, P = 101.325, Rh = 0.4490995); ``` -```jldoctest 1 -julia> outputs_sim = run!(models,meteo); +```julia +julia> outputs_sim = run!(mapping, meteo); ``` -```jldoctest 1 +```julia julia> outputs_sim[:var6] 1-element Vector{Float64}: 58.0138985 @@ -98,13 +99,13 @@ julia> outputs_sim[:var6] If we want to use special types for the variables, we can use the `type_promotion` argument: -```jldoctest 1 -julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0, var2=0.3), type_promotion = ModelMapping(Float64 => Float32)); +```julia +julia> models = SingleScaleModelSet(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0, var2=0.3), type_promotion = Dict(Float64 => Float32)); ``` We used `type_promotion` to force the status into Float32: -```jldoctest 1 +```julia julia> [typeof(models[i][1]) for i in keys(status(models))] 6-element Vector{DataType}: Float32 @@ -120,13 +121,13 @@ were converted to Float32, the two other variables that we gave were not convert because we want to give the ability to users to give any type for the variables they provide in the status. If we want all variables to be converted to Float32, we can pass them as Float32: -```jldoctest 1 -julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0f0, var2=0.3f0), type_promotion = Dict(Float64 => Float32)); +```julia +julia> models = SingleScaleModelSet(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=(var1=15.0f0, var2=0.3f0), type_promotion = Dict(Float64 => Float32)); ``` We used `type_promotion` to force the status into Float32: -```jldoctest 1 +```julia julia> [typeof(models[i][1]) for i in keys(status(models))] 6-element Vector{DataType}: Float32 @@ -137,19 +138,19 @@ julia> [typeof(models[i][1]) for i in keys(status(models))] Float32 ``` """ -struct ModelList{M<:NamedTuple,S} +struct SingleScaleModelSet{M<:NamedTuple,S} models::M status::S type_promotion::Union{Nothing,Dict} dependency_graph::DependencyGraph end -#=function ModelList(models::M, status::Status) where {M<:NamedTuple{names,T} where {names,T<:NTuple{N,<:AbstractModel} where {N}}} - ModelList(models, status) +#=function SingleScaleModelSet(models::M, status::Status) where {M<:NamedTuple{names,T} where {names,T<:NTuple{N,<:AbstractModel} where {N}}} + SingleScaleModelSet(models, status) end=# # General interface: -function ModelList( +function SingleScaleModelSet( args...; status=nothing, type_promotion::Union{Nothing,Dict}=nothing, @@ -179,7 +180,7 @@ function ModelList( ts_kwargs = homogeneous_ts_kwargs(status) ts_kwargs = add_model_vars(ts_kwargs, mods, type_promotion) - model_list = ModelList( + model_list = SingleScaleModelSet( mods, ts_kwargs, type_promotion, @@ -190,7 +191,7 @@ function ModelList( return model_list end -outputs(m::ModelList) = m.outputs +outputs(m::SingleScaleModelSet) = m.outputs parse_models(m) = NamedTuple([process(i) => i for i in m]) @@ -286,10 +287,10 @@ function homogeneous_ts_kwargs(kwargs::NamedTuple{N,T}) where {N,T} end """ - Base.copy(l::ModelList) - Base.copy(l::ModelList, status) + Base.copy(l::SingleScaleModelSet) + Base.copy(l::SingleScaleModelSet, status) -Copy a [`ModelList`](@ref), eventually with new values for the status. +Copy a [`SingleScaleModelSet`](@ref), eventually with new values for the status. # Examples @@ -300,7 +301,7 @@ using PlantSimEngine using PlantSimEngine.Examples; # Create a model list: -models = ModelList( +models = SingleScaleModelSet( process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), @@ -314,8 +315,8 @@ ml2 = copy(models) ml3 = copy(models, TimeStepTable([Status(var1=20.0, var2=0.5))]) ``` """ -function Base.copy(m::T) where {T<:ModelList} - ModelList( +function Base.copy(m::T) where {T<:SingleScaleModelSet} + SingleScaleModelSet( m.models, deepcopy(m.status), deepcopy(m.type_promotion), @@ -323,8 +324,8 @@ function Base.copy(m::T) where {T<:ModelList} ) end -function Base.copy(m::T, status) where {T<:ModelList} - ModelList( +function Base.copy(m::T, status) where {T<:SingleScaleModelSet} + SingleScaleModelSet( m.models, status, deepcopy(m.type_promotion), @@ -333,20 +334,20 @@ function Base.copy(m::T, status) where {T<:ModelList} end """ - Base.copy(l::AbstractArray{<:ModelList}) + Base.copy(l::AbstractArray{<:SingleScaleModelSet}) -Copy an array-alike of [`ModelList`](@ref) +Copy an array-alike of [`SingleScaleModelSet`](@ref) """ -function Base.copy(l::T) where {T<:AbstractArray{<:ModelList}} +function Base.copy(l::T) where {T<:AbstractArray{<:SingleScaleModelSet}} return [copy(i) for i in l] end """ - Base.copy(l::AbstractDict{N,<:ModelList} where N) + Base.copy(l::AbstractDict{N,<:SingleScaleModelSet} where N) -Copy a Dict-alike [`ModelList`](@ref) +Copy a Dict-alike [`SingleScaleModelSet`](@ref) """ -function Base.copy(l::T) where {T<:AbstractDict{N,<:ModelList} where {N}} +function Base.copy(l::T) where {T<:AbstractDict{N,<:SingleScaleModelSet} where {N}} return Dict([k => v for (k, v) in l]) end @@ -459,13 +460,13 @@ function convert_vars!(mapped_vars::Dict{Symbol,Dict{Symbol,Any}}, type_promotio end end -function Base.show(io::IO, m::MIME"text/plain", t::ModelList) +function Base.show(io::IO, m::MIME"text/plain", t::SingleScaleModelSet) show(io, m, dep(t)) println(io, "") show(io, m, status(t)) end # Short form printing (e.g. inside another object) -function Base.show(io::IO, t::ModelList) - print(io, "ModelList", (; zip(keys(t.models), typeof.(values(t.models)))...)) +function Base.show(io::IO, t::SingleScaleModelSet) + print(io, "SingleScaleModelSet", (; zip(keys(t.models), typeof.(values(t.models)))...)) end diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index 60cc07ad5..bdff60078 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -3,7 +3,7 @@ Status type used to store the values of the variables during simulation. It is mainly used as the structure to store the variables in the `TimeStepRow` of a `TimeStepTable` (see -[`PlantMeteo.jl` docs](https://palmstudio.github.io/PlantMeteo.jl/stable/)) of a [`ModelList`](@ref). +[`PlantMeteo.jl` docs](https://palmstudio.github.io/PlantMeteo.jl/stable/)) of a [`SingleScaleModelSet`](@ref). Most of the code is taken from MasonProtter/MutableNamedTuples.jl, so `Status` is a MutableNamedTuples with a few modifications, so in essence, it is a stuct that stores a `NamedTuple` of the references to the values of the variables, which makes it mutable. @@ -67,13 +67,22 @@ function Status(nt::NamedTuple{names}) where {names} Status(NamedTuple{names}(Ref.(values(nt)))) end +_status_vars(status) = getfield(status, :vars) +_status_values(status) = getindex.(values(_status_vars(status))) +_status_namedtuple(status) = NamedTuple{keys(status)}(values(status)) +_status_tuple(status) = values(status) +_status_iterate(status, iter=1) = iterate(NamedTuple(status), iter) +_status_firstindex(status) = 1 +_status_lastindex(status) = lastindex(NamedTuple(status)) +_status_indexed_iterate(status, i::Int, state=1) = Base.indexed_iterate(NamedTuple(status), i, state) + Base.keys(::Status{names}) where {names} = names -Base.values(st::Status) = getindex.(values(getfield(st, :vars))) +Base.values(st::Status) = _status_values(st) refvalues(mnt::Status) = values(getfield(mnt, :vars)) refvalue(mnt::Status, key::Symbol) = getfield(getfield(mnt, :vars), key) -Base.NamedTuple(mnt::Status) = NamedTuple{keys(mnt)}(values(mnt)) -Base.Tuple(mnt::Status) = values(mnt) +Base.NamedTuple(mnt::Status) = _status_namedtuple(mnt) +Base.Tuple(mnt::Status) = _status_tuple(mnt) function Base.show(io::IO, ::MIME"text/plain", t::Status) st_panel = Term.Panel( @@ -117,13 +126,13 @@ Base.propertynames(::Status{T,R}) where {T,R} = T Base.length(mnt::Status) = length(getfield(mnt, :vars)) Base.eltype(::Type{Status{N,T}}) where {N,T} = eltype.(eltype(T)) -Base.iterate(mnt::Status, iter=1) = iterate(NamedTuple(mnt), iter) +Base.iterate(mnt::Status, iter=1) = _status_iterate(mnt, iter) -Base.firstindex(mnt::Status) = 1 -Base.lastindex(mnt::Status) = lastindex(NamedTuple(mnt)) +Base.firstindex(mnt::Status) = _status_firstindex(mnt) +Base.lastindex(mnt::Status) = _status_lastindex(mnt) function Base.indexed_iterate(mnt::Status, i::Int, state=1) - Base.indexed_iterate(NamedTuple(mnt), i, state) + _status_indexed_iterate(mnt, i, state) end function Base.:(==)(s1::Status, s2::Status) @@ -169,4 +178,4 @@ function get_status_vector_max_length(s::Status) end end return max_len -end \ No newline at end of file +end diff --git a/src/component_models/StatusView.jl b/src/component_models/StatusView.jl index 1d74e099e..ceb1319fb 100644 --- a/src/component_models/StatusView.jl +++ b/src/component_models/StatusView.jl @@ -5,7 +5,7 @@ An equivalent of the `Status` struct, but with views instead of Refs. Allows to other data structures present elsewhere, and that we want to update on mutation. Like the `Status`, `StatusView` is used to store the values of the variables during a simulation, mainly as the structure to store the variables -in the `TimeStepRow` of a `TimeStepTable` (see [`PlantMeteo.jl` docs](https://palmstudio.github.io/PlantMeteo.jl/stable/)) of a [`ModelList`](@ref). +in the `TimeStepRow` of a `TimeStepTable` (see [`PlantMeteo.jl` docs](https://palmstudio.github.io/PlantMeteo.jl/stable/)) of a [`SingleScaleModelSet`](@ref). # Examples @@ -97,9 +97,9 @@ function Base.setindex!(s::StatusView, value, name) end Base.keys(::StatusView{names}) where {names} = names -Base.values(s::StatusView) = getindex.(values(getfield(s, :vars))) -Base.NamedTuple(mnt::StatusView) = NamedTuple{keys(mnt)}(values(mnt)) -Base.Tuple(mnt::StatusView) = values(mnt) +Base.values(s::StatusView) = _status_values(s) +Base.NamedTuple(mnt::StatusView) = _status_namedtuple(mnt) +Base.Tuple(mnt::StatusView) = _status_tuple(mnt) function Base.show(io::IO, s::StatusView) length(s) == 0 && return @@ -138,10 +138,10 @@ end Base.propertynames(::StatusView{T,R}) where {T,R} = T Base.length(mnt::StatusView) = length(getfield(mnt, :vars)) Base.eltype(::Type{StatusView{T}}) where {T} = T -Base.iterate(mnt::StatusView, iter=1) = iterate(NamedTuple(mnt), iter) -Base.firstindex(mnt::StatusView) = 1 -Base.lastindex(mnt::StatusView) = lastindex(NamedTuple(mnt)) +Base.iterate(mnt::StatusView, iter=1) = _status_iterate(mnt, iter) +Base.firstindex(mnt::StatusView) = _status_firstindex(mnt) +Base.lastindex(mnt::StatusView) = _status_lastindex(mnt) function Base.indexed_iterate(mnt::StatusView, i::Int, state=1) - Base.indexed_iterate(NamedTuple(mnt), i, state) -end \ No newline at end of file + _status_indexed_iterate(mnt, i, state) +end diff --git a/src/component_models/TimeStepTable.jl b/src/component_models/TimeStepTable.jl index d02253533..a7ce79516 100644 --- a/src/component_models/TimeStepTable.jl +++ b/src/component_models/TimeStepTable.jl @@ -7,7 +7,7 @@ from a `DataFrame`, but with each row being a `Status`. # Note -[`ModelList`](@ref) uses `TimeStepTable{Status}` by default (see examples below). +[`SingleScaleModelSet`](@ref) uses `TimeStepTable{Status}` by default (see examples below). # Examples @@ -25,7 +25,7 @@ TimeStepTable{Status}(df) # A leaf with several values for at least one of its variable will automatically use # TimeStepTable{Status} with the time steps: -models = ModelList( +models = SingleScaleModelSet( process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), diff --git a/src/component_models/get_status.jl b/src/component_models/get_status.jl index 55327614a..7b6bb0398 100644 --- a/src/component_models/get_status.jl +++ b/src/component_models/get_status.jl @@ -1,9 +1,9 @@ """ status(m) - status(m::AbstractArray{<:ModelList}) - status(m::AbstractDict{T,<:ModelList}) + status(m::AbstractArray{<:SingleScaleModelSet}) + status(m::AbstractDict{T,<:SingleScaleModelSet}) -Get a ModelList status, *i.e.* the state of the input (and output) variables. +Get a SingleScaleModelSet status, *i.e.* the state of the input (and output) variables. See also [`is_initialized`](@ref) and [`to_initialize`](@ref) @@ -15,7 +15,7 @@ using PlantSimEngine # Including example models and processes: using PlantSimEngine.Examples; -# Create a ModelList +# Create a SingleScaleModelSet models = ModelMapping( process1=Process1Model(1.0), process2=Process2Model(), @@ -67,8 +67,8 @@ function status(m, key::T) where {T<:Integer} end """ - getindex(component<:ModelList, key::Symbol) - getindex(component<:ModelList, key) + getindex(component<:SingleScaleModelSet, key::Symbol) + getindex(component<:SingleScaleModelSet, key) Indexing a component models structure: - with an integer, will return the status at the ith time-step @@ -79,7 +79,7 @@ Indexing a component models structure: ```julia using PlantSimEngine -lm = ModelList( +lm = SingleScaleModelSet( process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), @@ -95,10 +95,10 @@ lm[:var1][2] # Equivalent of the above 16.0 ``` """ -function Base.getindex(component::T, key) where {T<:ModelList} +function Base.getindex(component::T, key) where {T<:SingleScaleModelSet} status(component, key) end -function Base.setindex!(component::T, value, key) where {T<:ModelList} +function Base.setindex!(component::T, value, key) where {T<:SingleScaleModelSet} setproperty!(status(component), key, value) end diff --git a/src/dataframe.jl b/src/dataframe.jl index 2a9fba567..685279bb3 100644 --- a/src/dataframe.jl +++ b/src/dataframe.jl @@ -39,7 +39,7 @@ models = ModelMapping( df = DataFrame(models) ``` """ -function DataFrames.DataFrame(components::T) where {T<:AbstractArray{<:ModelMapping}} +function DataFrames.DataFrame(components::AbstractVector{<:ModelMapping}) df = DataFrame[] for (k, v) in enumerate(components) df_c = DataFrames.DataFrame(v) @@ -49,7 +49,7 @@ function DataFrames.DataFrame(components::T) where {T<:AbstractArray{<:ModelMapp reduce(vcat, df) end -function DataFrames.DataFrame(components::T) where {T<:AbstractDict{N,<:ModelMapping} where {N}} +function DataFrames.DataFrame(components::AbstractDict{<:Any,<:ModelMapping}) df = DataFrames.DataFrame[] for (k, v) in components df_c = DataFrames.DataFrame(v) @@ -64,6 +64,6 @@ end Implementation of `DataFrame` for a `ModelMapping` model with one time step. """ -function DataFrames.DataFrame(components::ModelMapping{T}) where {T} +function DataFrames.DataFrame(components::ModelMapping) DataFrames.DataFrame([NamedTuple(status(components)[1])]) end diff --git a/src/dependencies/dependencies.jl b/src/dependencies/dependencies.jl index d59031c17..4ff1e2b12 100644 --- a/src/dependencies/dependencies.jl +++ b/src/dependencies/dependencies.jl @@ -76,16 +76,17 @@ dep(;models...) function dep(nsteps=1; verbose::Bool=true, vars...) hard_dep = hard_dependencies((; vars...), verbose=verbose) deps = soft_dependencies(hard_dep, nsteps) + apply_update_dependencies!(deps, _model_specs_for_dependency_updates((; vars...))) # Return the dependency graph return deps end -function dep(m::ModelList) +function dep(m::SingleScaleModelSet) m.dependency_graph end -function dep!(m::ModelList, nsteps=1) +function dep!(m::SingleScaleModelSet, nsteps=1) traverse_dependency_graph!(m.dependency_graph; visit_hard_dep=false) do node if length(node.simulation_id) != nsteps node.simulation_id = fill(0, nsteps) @@ -114,6 +115,7 @@ function dep(mapping::AbstractDict{Symbol,T}; verbose::Bool=true) where {T} # nodes that have no soft-dependencies, and we set them as root nodes of the soft-dependency graph. The other nodes are set as children # of the nodes that they depend on. dep_graph = soft_dependencies_multiscale(soft_dep_graphs_roots, reverse_multiscale_mapping, hard_dep_dict) + apply_update_dependencies!(dep_graph, _model_specs_for_dependency_updates(mapping)) # During the building of the soft-dependency graph, we identified the inputs and outputs of each dependency node, # and also defined **inputs** as MappedVar if they are multiscale, i.e. if they take their values from another scale. # What we are missing is that we need to also define **outputs** as multiscale if they are needed by another scale. diff --git a/src/dependencies/dependency_graph.jl b/src/dependencies/dependency_graph.jl index 7063b77e9..1386169e8 100644 --- a/src/dependencies/dependency_graph.jl +++ b/src/dependencies/dependency_graph.jl @@ -1,5 +1,16 @@ abstract type AbstractDependencyNode end +""" + ProducerVariable(input, source) + +Dependency metadata for a producer variable that reaches a consumer input under +a different local name. +""" +struct ProducerVariable + input::Symbol + source::Symbol +end + mutable struct HardDependencyNode{T} <: AbstractDependencyNode value::T process::Symbol @@ -149,6 +160,10 @@ function _node_mapping(var_mapping::Pair{<:Union{AbstractString,Symbol},Symbol}) return SingleNodeMapping(first(var_mapping)), last(var_mapping) end +function _node_mapping(var_mapping::Pair{SameScale,Symbol}) + return first(var_mapping), last(var_mapping) +end + function _node_mapping(var_mapping) # Several organs are mapped to the variable: organ_mapped = MultiNodeMapping([first(i) for i in var_mapping]) diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index ba623980d..ccb7342d2 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -5,18 +5,27 @@ Compute the hard dependencies between models. """ +abstract type AbstractDomainDependencySelector end + +_is_domain_dependency_selector(x) = x isa AbstractDomainDependencySelector + function _normalize_hard_dependency_scales(scales, process::Symbol, dependency_process::Symbol) - if scales isa Symbol || scales isa AbstractString - return [Symbol(scales)] + if scales isa Symbol + return [scales] + elseif scales isa AbstractString + error( + "Invalid hard dependency scale declaration for process `$(process)` dependency `$(dependency_process)`: ", + "string scale names are removed. Use Symbol scales, e.g. `:Leaf`." + ) elseif scales isa Tuple || scales isa AbstractVector normalized = Symbol[] for s in scales - if s isa Symbol || s isa AbstractString - push!(normalized, Symbol(s)) + if s isa Symbol + push!(normalized, s) else error( "Invalid hard dependency scale declaration for process `$(process)` dependency `$(dependency_process)`: ", - "expected Symbol or String scales, got `$(typeof(s))`." + "expected Symbol scales, got `$(typeof(s))`." ) end end @@ -41,6 +50,7 @@ function hard_dependencies(models; scale=nothing, verbose::Bool=true) length(level_1_dep) == 0 && continue # if there is no dependency we skip the iteration dep_graph[process].dependency = level_1_dep for (p, depend) in pairs(level_1_dep) # for each dependency of the model i. p=:leaf_rank; depend=pairs(level_1_dep)[p] + _is_domain_dependency_selector(depend) && continue # The dependency can be given as multiscale, e.g. `leaf_area=AbstractLeaf_AreaModel => [m.leaf_symbol],` # This means we should search this model in another scale. This is not done here, but after the call to this # function in the other method for `hard_dependencies` below. @@ -51,7 +61,7 @@ function hard_dependencies(models; scale=nothing, verbose::Bool=true) push!(dep_not_found, p => (parent_process=process, type=first(depend), scales=target_scales)) continue else - # If we are not in a multi-scale setup e.g. in a ModelList, we shouldn't use a multiscale model. + # If we are not in a multi-scale setup, we shouldn't use a multiscale model. # But we still authorize it with a warning, and then proceed searching the dependency in this model list. verbose && @warn "Model $i has a multiscale hard dependency on $(first(depend)): $depend. Trying to find the model in this scale instead." depend = first(depend) @@ -59,7 +69,8 @@ function hard_dependencies(models; scale=nothing, verbose::Bool=true) end if hasproperty(models, p) - if typeof(getfield(models, p)) <: depend + candidate_model = getfield(models, p) + if model_(candidate_model) isa depend parent_dep = dep_graph[process] push!(parent_dep.children, dep_graph[p]) for child in parent_dep.children @@ -68,7 +79,7 @@ function hard_dependencies(models; scale=nothing, verbose::Bool=true) else if verbose @info string( - "Model ", typeof(i).name.name, " from process ", process, + "Model ", typeof(model_(i)).name.name, " from process ", process, isnothing(scale) ? "" : " at scale $scale", " needs a model that is a subtype of ", depend, " in process ", p @@ -86,10 +97,10 @@ function hard_dependencies(models; scale=nothing, verbose::Bool=true) else if verbose @info string( - "Model ", typeof(i).name.name, " from process ", process, + "Model ", typeof(model_(i)).name.name, " from process ", process, isnothing(scale) ? "" : " at scale $scale", " needs a model that is a subtype of ", depend, " in process ", - p, ", but the process is not parameterized in the ModelList." + p, ", but the process is not parameterized in the ModelMapping." ) end push!(dep_not_found, p => depend) @@ -214,8 +225,8 @@ function hard_dependencies(mapping::AbstractDict{Symbol,T}; verbose::Bool=true) end dep_node_model = only(dep_node_model) - if !isa(dep_node_model.value, model_type) - error("Model `$(typeof(parent_node.value))` from scale $organ requires a model of type `$model_type` at scale $s as a hard dependency, but the model found for this process is of type $(typeof(dep_node_model.value)).") + if !(model_(dep_node_model.value) isa model_type) + error("Model `$(typeof(model_(parent_node.value)))` from scale $organ requires a model of type `$model_type` at scale $s as a hard dependency, but the model found for this process is of type $(typeof(model_(dep_node_model.value))).") end # We make a new node out of the previous one: diff --git a/src/dependencies/is_graph_cyclic.jl b/src/dependencies/is_graph_cyclic.jl index 918eb28b7..1bfcd0e28 100644 --- a/src/dependencies/is_graph_cyclic.jl +++ b/src/dependencies/is_graph_cyclic.jl @@ -12,19 +12,16 @@ Check if the dependency graph is cyclic. Return a boolean indicating if the graph is cyclic, and the stack of nodes as a vector. """ function is_graph_cyclic(dependency_graph::DependencyGraph; full_stack=false, warn=true) - visited = Dict{Pair{AbstractModel,Symbol},Bool}() - recursion_stack = Dict{Pair{AbstractModel,Symbol},Bool}() - for node in values(dependency_graph.roots) - visited[node.value=>node.scale] = false - recursion_stack[node.value=>node.scale] = false - end + visited = IdDict{Any,Bool}() + recursion_stack = IdDict{Any,Bool}() for (root, node) in dependency_graph.roots - cycle_vec = Vector{Pair{AbstractModel,Symbol}}() - if is_graph_cyclic_(node, visited, recursion_stack, cycle_vec) + cycle_nodes = Any[] + if is_graph_cyclic_(node, visited, recursion_stack, cycle_nodes) + cycle_vec = _cycle_report_nodes(cycle_nodes) if full_stack - push!(cycle_vec, node.value => node.scale) + push!(cycle_vec, _cycle_report_node(node)) else # Keep just the cycle (the first node in the vector is the one that makes a cycle, we just detect the second time it happens on the stack): cycled_nodes = findall(x -> x == cycle_vec[1], cycle_vec) @@ -41,25 +38,26 @@ function is_graph_cyclic(dependency_graph::DependencyGraph; full_stack=false, wa end function is_graph_cyclic_(node, visited, recursion_stack, cycle_vec) - node_id = node.value => node.scale - visited[node_id] = true - recursion_stack[node_id] = true + visited[node] = true + recursion_stack[node] = true for child in node.children - child_id = child.value => child.scale - if !haskey(visited, child_id) && is_graph_cyclic_(child, visited, recursion_stack, cycle_vec) - push!(cycle_vec, child_id) + if !haskey(visited, child) && is_graph_cyclic_(child, visited, recursion_stack, cycle_vec) + push!(cycle_vec, child) return true - elseif haskey(recursion_stack, child_id) && recursion_stack[child_id] - push!(cycle_vec, child_id) + elseif get(recursion_stack, child, false) + push!(cycle_vec, child) return true end end - recursion_stack[node_id] = false + recursion_stack[node] = false return false end +_cycle_report_node(node) = node.value => node.scale +_cycle_report_nodes(nodes) = Pair{Any,Symbol}[_cycle_report_node(node) for node in nodes] + function print_cycle(cycle_vec) printed_cycle = Any[Term.RenderableText(string("{bold red}", last(cycle_vec[1]), ": ", typeof(first(cycle_vec[1]))))] leading_space = [1] diff --git a/src/dependencies/soft_dependencies.jl b/src/dependencies/soft_dependencies.jl index ecf36a8bd..cf203e5eb 100644 --- a/src/dependencies/soft_dependencies.jl +++ b/src/dependencies/soft_dependencies.jl @@ -18,7 +18,7 @@ using PlantSimEngine using PlantSimEngine.Examples # Create a model list: -models = ModelList( +models = SingleScaleModelSet( process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), @@ -139,6 +139,66 @@ function soft_dependencies(d::DependencyGraph{Dict{Symbol,HardDependencyNode}}, end # For multiscale mapping: +function _owning_soft_dependency_node(node::AbstractDependencyNode) + owner = node + depth = 0 + while !(owner isa SoftDependencyNode) + depth += 1 + depth <= 50 || error( + "Could not resolve the owning soft dependency for nested hard dependency ", + "`$(node.process)` at scale `$(node.scale)`." + ) + owner.parent === nothing && error( + "Nested hard dependency `$(node.process)` at scale `$(node.scale)` has no owning soft dependency." + ) + owner = owner.parent + end + return owner +end + +function _hard_dependency_candidates(parent_process::Symbol, target_scale::Symbol, hard_dep_dict::Dict{Pair{Symbol,Symbol},HardDependencyNode}) + scale_matches = HardDependencyNode[] + process_matches = HardDependencyNode[] + for ((hd_process, hd_scale), hd_node) in hard_dep_dict + hd_process == parent_process || continue + push!(process_matches, hd_node) + (hd_scale == target_scale || hd_node.scale == target_scale) && push!(scale_matches, hd_node) + end + return isempty(scale_matches) ? process_matches : scale_matches +end + +function _soft_graph_at_scale(soft_dep_graphs_roots::DependencyGraph{Dict{Symbol,Any}}, scale::Symbol) + haskey(soft_dep_graphs_roots.roots, scale) || error("Scale `$scale` not found while resolving soft dependency parent.") + return soft_dep_graphs_roots.roots[scale][:soft_dep_graph] +end + +function _resolve_soft_parent_node( + soft_dep_graphs_roots::DependencyGraph{Dict{Symbol,Any}}, + target_scale::Symbol, + parent_process::Symbol, + hard_dep_dict::Dict{Pair{Symbol,Symbol},HardDependencyNode} +) + roots_at_target_scale = _soft_graph_at_scale(soft_dep_graphs_roots, target_scale) + haskey(roots_at_target_scale, parent_process) && return roots_at_target_scale[parent_process] + + candidates = _hard_dependency_candidates(parent_process, target_scale, hard_dep_dict) + isempty(candidates) && error( + "Parent process `$parent_process` at scale `$target_scale` is not located in soft roots or nested hard dependencies." + ) + length(candidates) == 1 || error( + "Parent process `$parent_process` is an ambiguous nested hard dependency for scale `$target_scale`. ", + "Matching hard-dependency scales: $(Tuple((candidate.scale for candidate in candidates)))." + ) + + owner = _owning_soft_dependency_node(only(candidates)) + owner_graph = _soft_graph_at_scale(soft_dep_graphs_roots, owner.scale) + haskey(owner_graph, owner.process) || error( + "Owning soft dependency `$(owner.process)` at scale `$(owner.scale)` was resolved for nested hard dependency ", + "`$parent_process`, but it is not present in the finalized soft graph." + ) + return owner_graph[owner.process] +end + function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dict{Symbol,Any}}, reverse_multiscale_mapping, hard_dep_dict::Dict{Pair{Symbol,Symbol},HardDependencyNode}) independant_process_root = Dict{Pair{Symbol,Symbol},SoftDependencyNode}() @@ -171,33 +231,7 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic # and we need to add its parent(s) to the node, and the node as a child for (parent_soft_dep, soft_dep_vars) in pairs(soft_deps_not_hard) - # if the parent isn't registered as a soft dependency, it likely means the soft dependecy should be to an internal hard dependency to the parent - if (!haskey(soft_dep_graph, parent_soft_dep)) - - roots_at_given_scale = soft_dep_graphs_roots.roots[i.scale][:soft_dep_graph] - if !(parent_soft_dep in keys(roots_at_given_scale)) - master_node = () - for ((hd_key, hd_scale), hd) in hard_dep_dict - if parent_soft_dep == hd_key - master_node = hd - depth = 0 - # A cleaner way of preventing cycles or infinite loops would be more desirable - while !isa(master_node, SoftDependencyNode) && depth < 50 - master_node.parent === nothing && error("Finalised hard dependency has no parent") - master_node = master_node.parent - depth += 1 - end - - break - end - end - master_node == () && error("Parent is not located in hard deps, nor in roots, which should be the case when initalizing soft dependencies") - end - # NOTE : this may need to be propagated within internal hard dependencies' ancestors of this model... ? - parent_node = soft_dep_graphs_roots.roots[master_node.scale][:soft_dep_graph][master_node.process] - else - parent_node = soft_dep_graph[parent_soft_dep] - end + parent_node = _resolve_soft_parent_node(soft_dep_graphs_roots, i.scale, parent_soft_dep, hard_dep_dict) @@ -248,36 +282,7 @@ function soft_dependencies_multiscale(soft_dep_graphs_roots::DependencyGraph{Dic for org in keys(soft_deps_multiscale) for (parent_soft_dep, soft_dep_vars) in soft_deps_multiscale[org] - # if the node has a soft dependency on a node that is a nested hard dependency, - # have it point to the master node of that hard dependency instead of the internal node - # This check is meant in case the organ at the inspected scale is part of a hard dependency, - # and therefore already absent from the roots - - roots_at_given_scale = soft_dep_graphs_roots.roots[org][:soft_dep_graph] - if !(parent_soft_dep in keys(roots_at_given_scale)) - master_node = () - for ((hd_key, hd_scale), hd) in hard_dep_dict - if parent_soft_dep == hd_key - master_node = hd - depth = 0 - # A cleaner way of preventing cycles or infinite loops would be more desirable - while !isa(master_node, SoftDependencyNode) && depth < 50 - master_node.parent === nothing && error("Finalised hard dependency has no parent") - master_node = master_node.parent - depth += 1 - end - - break - end - end - - master_node == () && error("Parent is not located in hard deps, nor in roots, which should be the case when initalizing soft dependencies") - - # NOTE : this may need to be propagated within internal hard dependencies' ancestors of this model... ? - parent_node = soft_dep_graphs_roots.roots[master_node.scale][:soft_dep_graph][master_node.process] - else - parent_node = soft_dep_graphs_roots.roots[org][:soft_dep_graph][parent_soft_dep] - end + parent_node = _resolve_soft_parent_node(soft_dep_graphs_roots, org, parent_soft_dep, hard_dep_dict) # preventing a cyclic dependency: if the parent also has a dependency on the current node: if parent_node.parent !== nothing && any([i == p for p in parent_node.parent]) @@ -466,18 +471,20 @@ function search_inputs_in_multiscale_output(process, organ, inputs, soft_dep_gra # proc, organ, ins, soft_dep_graphs=soft_dep_graphs_roots.roots vars_input = flatten_vars(inputs[process]) - inputs_as_output_of_other_scale = Dict{Symbol,Dict{Symbol,Vector{Symbol}}}() + inputs_as_output_of_other_scale = Dict{Symbol,Dict{Symbol,Vector{ProducerVariable}}}() for (var, val) in pairs(vars_input) # e.g. var = :leaf_surfaces;val = vars_input[var] # The variable is a multiscale variable: if isa(val, MappedVar) var_organ = mapped_organ(val) - (isnothing(var_organ) || var_organ == Symbol("")) && continue # If the variable maps to nothing we skip it (e.g. [PreviousTimeStep(:var1)] or [:var => :new_var]) + isnothing(var_organ) && continue # If the variable maps to nothing we skip it (e.g. [PreviousTimeStep(:var1)] or [:var => :new_var]) if !isa(var_organ, AbstractVector) # In case the organ is given as a singleton (e.g. :Soil instead of [:Soil]) var_organ = [var_organ] end - @assert all(var_o != organ for var_o in var_organ) "$var in process $process is set to be multiscale, but points to its own scale ($organ). This is not allowed." + if !all(var_o != organ for var_o in var_organ) + error("$var in process $process is set to be multiscale, but points to its own scale ($organ). This is not allowed.") + end for org in var_organ # e.g. org = :Leaf # The variable is a multiscale variable: haskey(soft_dep_graphs, org) || error("Scale $org not found in the mapping, but mapped to the $organ scale.") @@ -511,6 +518,7 @@ end function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, organ_source, variable, value) + producer_var = ProducerVariable(value, variable) for (proc_output, pairs_vars_output) in soft_dep_graphs[organ_source][:outputs] # e.g. proc_output = :maintenance_respiration; pairs_vars_output = soft_dep_graphs_roots.roots[organ_source][:outputs][proc_output] vars_output = flatten_vars(pairs_vars_output) @@ -519,12 +527,12 @@ function add_input_as_output!(inputs_as_output_of_other_scale, soft_dep_graphs, # The variable is found at another scale: if haskey(inputs_as_output_of_other_scale, organ_source) if haskey(inputs_as_output_of_other_scale[organ_source], proc_output) - push!(inputs_as_output_of_other_scale[organ_source][proc_output], value) + push!(inputs_as_output_of_other_scale[organ_source][proc_output], producer_var) else - inputs_as_output_of_other_scale[organ_source][proc_output] = [value] + inputs_as_output_of_other_scale[organ_source][proc_output] = [producer_var] end else - inputs_as_output_of_other_scale[organ_source] = Dict(proc_output => [value]) + inputs_as_output_of_other_scale[organ_source] = Dict(proc_output => [producer_var]) end end end diff --git a/src/dependencies/update_dependencies.jl b/src/dependencies/update_dependencies.jl new file mode 100644 index 000000000..4d6f74cec --- /dev/null +++ b/src/dependencies/update_dependencies.jl @@ -0,0 +1,189 @@ +function _model_specs_for_dependency_updates(vars::NamedTuple) + return Dict( + :Default => Dict{Symbol,ModelSpec}( + process(model) => as_model_spec(model) for model in values(vars) + ) + ) +end + +_model_specs_for_dependency_updates(mapping::AbstractDict{Symbol,T}) where {T} = + Dict(scale => parse_model_specs(declarations) for (scale, declarations) in pairs(mapping)) + +function _update_variables_for_spec(spec::ModelSpec) + vars = Set{Symbol}() + for update in updates(spec) + union!(vars, update.variables) + end + return vars +end + +function _update_after_for_var(spec::ModelSpec, var::Symbol) + after = Symbol[] + for update in updates(spec) + var in update.variables || continue + append!(after, update.after) + end + return unique(after) +end + +function _canonical_output_vars(spec::ModelSpec) + vars = Symbol[] + routing = output_routing(spec) + for var in keys(outputs_(spec)) + mode = var in keys(routing) ? routing[var] : :canonical + mode == :stream_only && continue + push!(vars, var) + end + return vars +end + +function _validate_updates_for_scale(scale::Symbol, specs_at_scale::Dict{Symbol,ModelSpec}, ignored_processes::Set{Symbol}=Set{Symbol}()) + canonical_writers = Dict{Symbol,Vector{Symbol}}() + + for (process, spec) in specs_at_scale + model_outputs = Set(keys(outputs_(spec))) + for update in updates(spec) + isempty(update.after) && error( + "Updates declaration for process `$(process)` at scale `$(scale)` must specify `after=...`. ", + "Use for example `Updates(:var; after=:producer_process)`." + ) + for var in update.variables + var in model_outputs || error( + "Updates declaration for process `$(process)` at scale `$(scale)` mentions variable `$(var)`, ", + "but this model does not output `$(var)`." + ) + for after_process in update.after + haskey(specs_at_scale, after_process) || error( + "Updates declaration for variable `$(var)` in process `$(process)` at scale `$(scale)` ", + "references unknown process `$(after_process)`." + ) + var in keys(outputs_(specs_at_scale[after_process])) || error( + "Updates declaration `Updates(:$(var); after=:$(after_process))` in process `$(process)` ", + "at scale `$(scale)` requires `$(after_process)` to output `$(var)`." + ) + end + end + end + + process in ignored_processes && continue + for var in _canonical_output_vars(spec) + push!(get!(canonical_writers, var, Symbol[]), process) + end + end + + for (var, writers) in canonical_writers + length(writers) <= 1 && continue + updater_flags = Dict(process => (var in _update_variables_for_spec(specs_at_scale[process])) for process in writers) + primary_writers = [process for process in writers if !updater_flags[process]] + update_writers = [process for process in writers if updater_flags[process]] + + length(primary_writers) == 1 || error( + "Ambiguous canonical writers for variable `$(var)` at scale `$(scale)`: ", + join(writers, ", "), + ". Keep one primary writer and declare additional writers with `Updates(:$(var); after=:primary_process)`." + ) + + primary = only(primary_writers) + for updater in update_writers + after = _update_after_for_var(specs_at_scale[updater], var) + primary in after || error( + "Update writer `$(updater)` for variable `$(var)` at scale `$(scale)` must declare its primary producer in `after`. ", + "Use `Updates(:$(var); after=:$(primary))` or include `:$(primary)` in the `after` list." + ) + end + + for i in eachindex(update_writers), j in (i+1):length(update_writers) + left = update_writers[i] + right = update_writers[j] + left_after = _update_after_for_var(specs_at_scale[left], var) + right_after = _update_after_for_var(specs_at_scale[right], var) + (left in right_after || right in left_after) || error( + "Update writers `$(left)` and `$(right)` both update variable `$(var)` at scale `$(scale)` ", + "without an ordering relation. Declare one updater `after` the other." + ) + end + end + + return nothing +end + +function validate_update_dependencies( + model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}; + ignored_processes_by_scale::Dict{Symbol,Set{Symbol}}=Dict{Symbol,Set{Symbol}}() +) + for (scale, specs_at_scale) in model_specs + _validate_updates_for_scale(scale, specs_at_scale, get(ignored_processes_by_scale, scale, Set{Symbol}())) + end + return nothing +end + +function _soft_nodes_by_scale_process(graph::DependencyGraph) + nodes = Dict{Tuple{Symbol,Symbol},SoftDependencyNode}() + for node in traverse_dependency_graph(graph, false) + nodes[(node.scale, node.process)] = node + end + return nodes +end + +function _delete_root_for_node!(graph::DependencyGraph, node::SoftDependencyNode) + if haskey(graph.roots, node.process) + delete!(graph.roots, node.process) + end + key = node.scale => node.process + if haskey(graph.roots, key) + delete!(graph.roots, key) + end + return nothing +end + +function _add_update_edge!(graph::DependencyGraph, parent_node::SoftDependencyNode, child_node::SoftDependencyNode) + parent_node === child_node && error( + "Invalid Updates dependency: process `$(child_node.process)` cannot be ordered after itself." + ) + + child_node in parent_node.children || push!(parent_node.children, child_node) + if child_node.parent === nothing + child_node.parent = [parent_node] + elseif !(parent_node in child_node.parent) + push!(child_node.parent, parent_node) + end + _delete_root_for_node!(graph, child_node) + return nothing +end + +function apply_update_dependencies!(graph::DependencyGraph, model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}) + validate_hard_dependency_timestep_consistency(model_specs, graph) + ignored_processes_by_scale = _hard_dependency_children(graph) + validate_update_dependencies(model_specs; ignored_processes_by_scale=ignored_processes_by_scale) + has_updates = any(!isempty(updates(spec)) for specs_at_scale in values(model_specs) for spec in values(specs_at_scale)) + has_updates || return graph + + nodes = _soft_nodes_by_scale_process(graph) + + for (scale, specs_at_scale) in model_specs + ignored_processes = get(ignored_processes_by_scale, scale, Set{Symbol}()) + for (process, spec) in specs_at_scale + process in ignored_processes && continue + child_node = get(nodes, (scale, process), nothing) + isnothing(child_node) && continue + for update in updates(spec) + for after_process in update.after + parent_node = get(nodes, (scale, after_process), nothing) + isnothing(parent_node) && error( + "Updates declaration for process `$(process)` at scale `$(scale)` references `$(after_process)`, ", + "but that process is not an executable dependency node. It may be nested as a hard dependency." + ) + _add_update_edge!(graph, parent_node, child_node) + end + end + end + end + + iscyclic, cycle_vec = is_graph_cyclic(graph; warn=false) + iscyclic && error( + "Cyclic dependency detected after applying Updates declarations. Cycle: \n", + print_cycle(cycle_vec) + ) + + return graph +end diff --git a/src/domains/domain_run_loops.jl b/src/domains/domain_run_loops.jl new file mode 100644 index 000000000..bf08bcd02 --- /dev/null +++ b/src/domains/domain_run_loops.jl @@ -0,0 +1,89 @@ +function _run_single_status_domain_simulation!( + simulation::DomainSimulation, + constants, + nsteps::Int +) + run_order = _domain_run_order(simulation) + for i in 1:nsteps + for domain in run_order + _materialize_routes_for_domain!(simulation, domain, i) + _run_domain_models!(simulation, domain, constants, i) + _update_domain_environment_index!(simulation, domain) + end + for domain in run_order + domain.kind == :scene && continue + _domain_has_post_scene_work(simulation, domain) || continue + _materialize_routes_for_domain!(simulation, domain, i) + _run_domain_models!(simulation, domain, constants, i; phase=:post_scene) + _update_domain_environment_index!(simulation, domain) + end + end + return simulation +end + +function _run_staged_graph_domain_simulation!( + simulation::DomainSimulation, + object, + raw_meteo, + constants, + nsteps::Int; + check=true, + executor=SequentialEx(), + type_promotion=nothing, +) + run_order = _domain_run_order(simulation) + graph_runtimes = Dict{Symbol,DomainGraphRuntime}() + + for step in 1:nsteps + for scene_phase in (false, true) + for domain in run_order + (domain.kind == :scene) == scene_phase || continue + if _is_graph_domain(domain) + domain.kind == :scene && error( + "Scene domain `$(domain.name)` is MTG-backed. The MTG-domain runner currently supports ", + "single-status scene domains only." + ) + runtime = get(graph_runtimes, domain.name, nothing) + if isnothing(runtime) + domain_type_promotion = isnothing(type_promotion) ? _type_promotion(_domain_mapping(domain)) : type_promotion + runtime = _prepare_graph_domain_runtime!( + simulation, + domain, + object, + raw_meteo, + constants, + nsteps; + check=check, + executor=executor, + type_promotion=domain_type_promotion, + ) + graph_runtimes[domain.name] = runtime + end + _run_graph_domain_step!(simulation, domain, runtime, step, nsteps; check=check) + else + _materialize_routes_for_domain!(simulation, domain, step) + _run_domain_models!(simulation, domain, constants, step) + _update_domain_environment_index!(simulation, domain) + end + end + end + for domain in run_order + domain.kind == :scene && continue + _domain_has_post_scene_work(simulation, domain) || continue + if _is_graph_domain(domain) + runtime = graph_runtimes[domain.name] + _run_graph_domain_step!(simulation, domain, runtime, step, nsteps; check=check, phase=:post_scene) + else + _materialize_routes_for_domain!(simulation, domain, step) + _run_domain_models!(simulation, domain, constants, step; phase=:post_scene) + _update_domain_environment_index!(simulation, domain) + end + end + end + + for runtime in values(graph_runtimes) + _finalize_graph_domain_runtime!(runtime) + end + + return simulation +end diff --git a/src/domains/domain_scheduler.jl b/src/domains/domain_scheduler.jl new file mode 100644 index 000000000..3bdff2680 --- /dev/null +++ b/src/domains/domain_scheduler.jl @@ -0,0 +1,167 @@ +function _domain_order_edges(mapping::SimulationMapping, route_bindings=nothing) + edges = Dict(domain.name => Set{Symbol}() for domain in mapping.domains) + non_scene_domains = [domain.name for domain in mapping.domains if domain.kind != :scene] + scene_domains = [domain.name for domain in mapping.domains if domain.kind == :scene] + for source in non_scene_domains, target in scene_domains + source == target || push!(edges[source], target) + end + + if !isnothing(route_bindings) + for (i, route) in enumerate(mapping.routes) + target = route.to.domain + for producer in route_bindings[i] + producer.domain == target && continue + push!(edges[producer.domain], target) + end + end + end + + return edges +end + +function _domain_run_order(mapping::SimulationMapping, route_bindings=nothing) + domains_by_name = Dict(domain.name => domain for domain in mapping.domains) + declaration_index = Dict(domain.name => i for (i, domain) in enumerate(mapping.domains)) + edges = _domain_order_edges(mapping, route_bindings) + indegree = Dict(domain.name => 0 for domain in mapping.domains) + for targets in values(edges), target in targets + indegree[target] = get(indegree, target, 0) + 1 + end + + ready = sort!( + [name for (name, degree) in indegree if degree == 0]; + by=name -> declaration_index[name], + ) + ordered = Symbol[] + while !isempty(ready) + current = popfirst!(ready) + push!(ordered, current) + for target in sort!(collect(edges[current]); by=name -> declaration_index[name]) + indegree[target] -= 1 + if indegree[target] == 0 + push!(ready, target) + sort!(ready; by=name -> declaration_index[name]) + end + end + end + + if length(ordered) != length(mapping.domains) + cyclic = sort!( + [name for (name, degree) in indegree if degree > 0]; + by=name -> declaration_index[name], + ) + error( + "Cyclic domain run-order constraints detected among domains: ", + join((":" * string(name) for name in cyclic), ", "), + ". Check route sources/targets and `kind=:scene` phase constraints." + ) + end + + return [domains_by_name[name] for name in ordered] +end + +function _domain_run_order(simulation::DomainSimulation) + return _domain_run_order(simulation.mapping, simulation.route_bindings) +end + +function _domain_node_due(simulation::DomainSimulation, domain::Domain, node::SoftDependencyNode, step::Int) + key = DomainModelKey(domain.name, node.scale, node.process) + clock = simulation.model_clocks[key] + return _should_run_at_time(clock, float(step)) +end + +function _hard_domain_dependency_keys(simulation::DomainSimulation) + keys = Set{DomainModelKey}() + for producers in values(simulation.hard_domain_dependency_bindings) + union!(keys, producers) + end + return keys +end + +function _is_hard_domain_dependency(simulation::DomainSimulation, key::DomainModelKey) + key in _hard_domain_dependency_keys(simulation) +end + +function _has_hard_domain_parent(simulation::DomainSimulation, domain::Domain, node::SoftDependencyNode) + AbstractTrees.isroot(node) && return false + hard_keys = _hard_domain_dependency_keys(simulation) + for parent in node.parent + parent_key = DomainModelKey(domain.name, parent.scale, parent.process) + parent_key in hard_keys && return true + end + return false +end + +function _phase_allows_hard_parent(phase::Symbol, has_hard_parent::Bool) + phase == :normal && return !has_hard_parent + phase == :post_scene && return has_hard_parent + error("Unknown domain scheduling phase `$(phase)`.") +end + +function _should_visit_domain_node( + simulation::DomainSimulation, + domain::Domain, + node::SoftDependencyNode; + phase::Symbol, +) + key = DomainModelKey(domain.name, node.scale, node.process) + _is_hard_domain_dependency(simulation, key) && return false + has_hard_parent = _has_hard_domain_parent(simulation, domain, node) + return _phase_allows_hard_parent(phase, has_hard_parent) +end + +function _has_hard_domain_parent(simulation::DomainSimulation, domain::Domain, key::DomainModelKey) + node = try + _find_dependency_node(simulation.dependency_graphs[domain.name], key) + catch + return false + end + return _has_hard_domain_parent(simulation, domain, node) +end + +function _has_domain_soft_node(simulation::DomainSimulation, domain::Domain, key::DomainModelKey) + try + _find_dependency_node(simulation.dependency_graphs[domain.name], key) + return true + catch + return false + end +end + +function _should_publish_domain_key( + simulation::DomainSimulation, + domain::Domain, + key::DomainModelKey; + phase::Symbol, +) + _has_domain_soft_node(simulation, domain, key) || return false + _is_hard_domain_dependency(simulation, key) && return false + has_hard_parent = _has_hard_domain_parent(simulation, domain, key) + return _phase_allows_hard_parent(phase, has_hard_parent) +end + +function _domain_has_post_scene_work(simulation::DomainSimulation, domain::Domain) + for key in keys(simulation.model_specs) + key.domain == domain.name || continue + _is_hard_domain_dependency(simulation, key) && continue + _has_hard_domain_parent(simulation, domain, key) && return true + end + return false +end + +function _domain_parents_ready( + simulation::DomainSimulation, + domain::Domain, + node::SoftDependencyNode, + step::Int, + ran::Set{DomainModelKey} +) + AbstractTrees.isroot(node) && return true + for parent in node.parent + _domain_node_due(simulation, domain, parent, step) || continue + parent_key = DomainModelKey(domain.name, parent.scale, parent.process) + _is_hard_domain_dependency(simulation, parent_key) && continue + parent_key in ran || return false + end + return true +end diff --git a/src/domains/domain_simulation.jl b/src/domains/domain_simulation.jl new file mode 100644 index 000000000..9f7d3654d --- /dev/null +++ b/src/domains/domain_simulation.jl @@ -0,0 +1,1290 @@ +""" + Domain(name, mapping; kind=:generic, selector=nothing) + Domain(name; kind, mapping, selector=nothing) + +Reusable model domain used by [`SimulationMapping`](@ref). + +Domains can wrap either a single-status `ModelMapping` or a scale-keyed +multiscale `ModelMapping` backed by an MTG subtree selected with `selector`. +The domain identity is kept alongside scale/process identity so multi-plant, +soil, scene, and environment domains can be compiled into one simulation +without renaming scales. +""" +struct Domain{M,S} + name::Symbol + kind::Symbol + mapping::M + selector::S +end + +function _normalize_domain_mapping(mapping) + mapping isa ModelMapping && return mapping + if mapping isa Tuple + status_index = findlast(x -> x isa Status, mapping) + if !isnothing(status_index) + status_index == length(mapping) || error( + "Raw tuple domain mappings may contain a positional Status only as the last element." + ) + return ModelMapping(mapping[begin:(end - 1)]...; status=last(mapping)) + end + return ModelMapping(mapping...) + end + return ModelMapping(mapping) +end + +function Domain(name::Union{Symbol,AbstractString}, mapping; kind::Union{Symbol,AbstractString}=:generic, selector=nothing) + normalized_mapping = copy(_normalize_domain_mapping(mapping)) + return Domain(Symbol(name), Symbol(kind), normalized_mapping, selector) +end + +function Domain(name::Union{Symbol,AbstractString}; kind::Union{Symbol,AbstractString}=:generic, mapping, selector=nothing) + return Domain(name, mapping; kind=kind, selector=selector) +end + +_domain_mapping(domain::Domain) = _normalize_domain_mapping(domain.mapping) + +""" + DomainModelKey(domain, scale, process) + +Stable key for one model process inside one domain and scale. +""" +struct DomainModelKey + domain::Symbol + scale::Symbol + process::Symbol +end + +Base.show(io::IO, key::DomainModelKey) = print(io, key.domain, "/", key.scale, "/", key.process) + +""" + AllDomains(; kind=nothing, domain=nothing, scale=nothing, process=nothing, var=nothing, policy=HoldLast()) + AllDomains(process; kwargs...) + +Value selector for scene-level models that intentionally consume output streams +from matching models in several domains. + +`policy` controls how producer streams are sampled when the consumer runs at a +different rate. `AllDomains` does not provide hard-dependency call-stack +control; use [`HardDomains`](@ref) when the parent model must manually run +the selected models. +""" +struct AllDomains{P<:SchedulePolicy} <: AbstractDomainDependencySelector + kind::Union{Nothing,Symbol} + domain::Union{Nothing,Symbol} + scale::Union{Nothing,Symbol} + process::Union{Nothing,Symbol} + var::Union{Nothing,Symbol} + policy::P +end + +function AllDomains(; kind=nothing, domain=nothing, scale=nothing, process=nothing, var=nothing, policy::SchedulePolicy=HoldLast()) + return AllDomains( + isnothing(kind) ? nothing : Symbol(kind), + isnothing(domain) ? nothing : Symbol(domain), + isnothing(scale) ? nothing : Symbol(scale), + isnothing(process) ? nothing : Symbol(process), + isnothing(var) ? nothing : Symbol(var), + policy, + ) +end + +AllDomains(process::Union{Symbol,AbstractString}; kwargs...) = AllDomains(; process=Symbol(process), kwargs...) + +""" + HardDomains(; kind=nothing, domain=nothing, scale=nothing, process=nothing) + HardDomains(process; kwargs...) + +Selector for models that are cross-domain hard dependencies. A model declaring +`dep(model) = (; name=HardDomains(...))` +can retrieve executable targets with [`dependency_targets`](@ref) and manually +execute them with [`run_target!`](@ref). +""" +struct HardDomains <: AbstractDomainDependencySelector + kind::Union{Nothing,Symbol} + domain::Union{Nothing,Symbol} + scale::Union{Nothing,Symbol} + process::Union{Nothing,Symbol} +end + +function HardDomains(; kind=nothing, domain=nothing, scale=nothing, process=nothing) + return HardDomains( + isnothing(kind) ? nothing : Symbol(kind), + isnothing(domain) ? nothing : Symbol(domain), + isnothing(scale) ? nothing : Symbol(scale), + isnothing(process) ? nothing : Symbol(process), + ) +end + +HardDomains(process::Union{Symbol,AbstractString}; kwargs...) = HardDomains(; process=Symbol(process), kwargs...) + +function _hard_domains_from_call_selector(selector::AbstractObjectMultiplicity) + c = criteria(selector) + unsupported = (:species, :name, :var, :relation, :within, :policy, :window) + any(key -> haskey(c, key) && !isnothing(getproperty(c, key)), unsupported) && error( + "Current `Calls(...)` hard-domain bridge only supports `kind`, `domain`, `scale`, and `process` selectors. ", + "Unsupported selector criteria: $(filter(key -> haskey(c, key), unsupported))." + ) + return HardDomains( + kind=haskey(c, :kind) ? c.kind : nothing, + domain=haskey(c, :domain) ? c.domain : nothing, + scale=haskey(c, :scale) ? c.scale : nothing, + process=haskey(c, :process) ? c.process : nothing, + ) +end + +function _model_spec_dependency_selector(dep_name::Symbol, selector::Call) + return _hard_domains_from_call_selector(selector.selector) +end + +function _model_spec_dependency_selector(dep_name::Symbol, selector::AbstractObjectMultiplicity) + return _hard_domains_from_call_selector(selector) +end + +function _push_selector_term!(terms::Vector{String}, name::Symbol, value) + isnothing(value) || push!(terms, "$(name)=:$(value)") + return terms +end + +function _policy_display(policy::SchedulePolicy) + return string(nameof(typeof(policy)), "()") +end + +function Base.show(io::IO, selector::AllDomains) + terms = String[] + _push_selector_term!(terms, :kind, selector.kind) + _push_selector_term!(terms, :domain, selector.domain) + _push_selector_term!(terms, :scale, selector.scale) + _push_selector_term!(terms, :process, selector.process) + _push_selector_term!(terms, :var, selector.var) + selector.policy isa HoldLast || push!(terms, "policy=$(_policy_display(selector.policy))") + print(io, "AllDomains(", join(terms, ", "), ")") +end + +function Base.show(io::IO, selector::HardDomains) + terms = String[] + _push_selector_term!(terms, :kind, selector.kind) + _push_selector_term!(terms, :domain, selector.domain) + _push_selector_term!(terms, :scale, selector.scale) + _push_selector_term!(terms, :process, selector.process) + print(io, "HardDomains(", join(terms, ", "), ")") +end + +include("routes.jl") + +""" + SimulationMapping(domains...) + +Top-level composition of reusable domains. This is the incremental entry point +for multi-plant/soil/scene simulations. +""" +struct SimulationMapping + domains::Vector{Domain} + routes::Vector{Route} +end + +function SimulationMapping(domains::Domain...; routes=()) + names = [domain.name for domain in domains] + duplicates = unique(filter(name -> count(==(name), names) > 1, names)) + isempty(duplicates) || error("Duplicate domain name(s) in SimulationMapping: $(join(duplicates, ", ")).") + normalized_routes = Route[] + for route in routes + route isa Route || error("SimulationMapping routes must be `Route` objects, got `$(typeof(route))`.") + push!(normalized_routes, route) + end + return SimulationMapping(collect(domains), normalized_routes) +end + +""" + DomainRunContext + +Runtime context passed as `extra` to models run by [`SimulationMapping`](@ref). +Scene models can call [`dependency_values`](@ref) to consume resolved +`AllDomains` value dependencies, or [`dependency_targets`](@ref) to manually +run resolved `HardDomains` dependencies. +""" +struct DomainRunContext + simulation + consumer::DomainModelKey + step::Int + clock::ClockSpec + constants +end + +struct ModelTarget + simulation + key + node + step::Int + model + models + status + meteo + constants + extra +end + +struct DomainNodeValues{I,T<:AbstractVector} + ids::I + values::T +end + +DomainNodeValues(values::AbstractVector) = DomainNodeValues(nothing, values) + +""" + model_target(model, models, status, meteo=nothing, constants=nothing, extra=nothing; kwargs...) + +Build one executable model target. This is the low-level representation used by +hard-domain dependencies and can also be used by ordinary same-status hard +dependencies. +""" +function model_target( + model, + models, + status, + meteo=nothing, + constants=nothing, + extra=nothing; + simulation=nothing, + key=nothing, + node=nothing, + step::Integer=1, +) + return ModelTarget(simulation, key, node, Int(step), model, models, status, meteo, constants, extra) +end + +function dependency_targets( + models, + status, + dependency_name::Symbol; + meteo=nothing, + constants=nothing, + extra=nothing, +) + hasproperty(models, dependency_name) || error( + "No model named `$(dependency_name)` is available in `models`. ", + "Declare the hard dependency with `dep(model)` and include a model for that process in the mapping." + ) + return ModelTarget[ + model_target(getproperty(models, dependency_name), models, status, meteo, constants, extra), + ] +end + +dependency_targets(models, status, dependency_name::AbstractString; kwargs...) = + dependency_targets(models, status, Symbol(dependency_name); kwargs...) + +dependency_target(args...; kwargs...) = only(dependency_targets(args...; kwargs...)) + +struct DomainGraphState{S<:AbstractVector} + simulations::S +end + +struct DomainGraphRuntime + state::DomainGraphState + meteo + constants + effective_multirate::Bool + timeline::TimelineContext + meteo_sampler + executor +end + +""" + DomainSimulation + +Result and mutable runtime state for a [`SimulationMapping`](@ref) run. +""" +mutable struct DomainSimulation + mapping::SimulationMapping + environment::AbstractEnvironmentBackend + model_specs::Dict{DomainModelKey,ModelSpec} + model_clocks::Dict{DomainModelKey,ClockSpec} + dependency_graphs::Dict{Symbol,DependencyGraph} + dependency_bindings::Dict{Tuple{DomainModelKey,Symbol},Vector{DomainModelKey}} + dependency_variables::Dict{Tuple{DomainModelKey,Symbol},Union{Nothing,Symbol}} + dependency_policies::Dict{Tuple{DomainModelKey,Symbol},SchedulePolicy} + hard_domain_dependency_bindings::Dict{Tuple{DomainModelKey,Symbol},Vector{DomainModelKey}} + route_bindings::Vector{Vector{DomainModelKey}} + route_clocks::Vector{ClockSpec} + streams::Dict{Tuple{DomainModelKey,Symbol},Vector{Pair{Int,Any}}} + outputs::Dict{Tuple{DomainModelKey,Symbol},Vector{Any}} + domain_states::Dict{Symbol,Any} + timeline::TimelineContext +end + +outputs(simulation::DomainSimulation) = simulation.outputs + +function status(state::DomainGraphState) + statuses = Dict{Symbol,Vector{Status}}() + for graph_simulation in state.simulations + for (scale, statuses_at_scale) in status(graph_simulation) + append!(get!(statuses, scale, Status[]), statuses_at_scale) + end + end + return statuses +end + +function _global_scale_statuses(simulation::DomainSimulation, scale::Symbol) + statuses = Status[] + for state in values(simulation.domain_states) + if state isa DomainGraphState + graph_statuses = status(state) + haskey(graph_statuses, scale) || continue + append!(statuses, graph_statuses[scale]) + elseif state isa ModelMapping{SingleScale} && scale == :Default + push!(statuses, status(state)) + end + end + return statuses +end + +function _domain_declares_scale(domain::Domain, scale::Symbol) + mapping = _domain_mapping(domain) + mapping isa ModelMapping{MultiScale} || return false + return any(entry -> first(entry) == scale, pairs(mapping)) +end + +function _simulation_declares_graph_scale(simulation::DomainSimulation, scale::Symbol) + return any(domain -> _domain_declares_scale(domain, scale), simulation.mapping.domains) +end + +function status(simulation::DomainSimulation, domain_name::Symbol) + state = get(simulation.domain_states, domain_name, nothing) + if isnothing(state) + statuses = _global_scale_statuses(simulation, domain_name) + isempty(statuses) && _simulation_declares_graph_scale(simulation, domain_name) && return statuses + isempty(statuses) && error( + "No domain named `$(domain_name)` and no global scale `$(domain_name)` exists in this DomainSimulation." + ) + return statuses + end + state isa ModelMapping{SingleScale} || error( + "Domain `$(domain_name)` is MTG-backed. Use `status(simulation, domain, scale)` to inspect statuses at one scale." + ) + return status(state) +end + +status(simulation::DomainSimulation, domain_name::AbstractString) = status(simulation, Symbol(domain_name)) + +function status(simulation::DomainSimulation, domain_name::Symbol, scale::Symbol) + state = get(simulation.domain_states, domain_name, nothing) + isnothing(state) && error("Domain `$(domain_name)` has no runtime state.") + state isa DomainGraphState || error( + "Domain `$(domain_name)` is single-status. Use `status(simulation, domain)` without a scale." + ) + graph_statuses = status(state) + haskey(graph_statuses, scale) && return graph_statuses[scale] + domain = _domain_for_name(simulation.mapping, domain_name) + _domain_declares_scale(domain, scale) && return Status[] + error("Domain `$(domain_name)` has no runtime statuses and no declared mapping at scale `$(scale)`.") +end + +status(simulation::DomainSimulation, domain_name::AbstractString, scale::Union{Symbol,AbstractString}) = + status(simulation, Symbol(domain_name), Symbol(scale)) + +function _domain_entries(domain::Domain) + mapping = _domain_mapping(domain) + return collect(pairs(mapping)) +end + +function _validate_initial_domain_mapping(domain::Domain) + mapping = _domain_mapping(domain) + mapping isa ModelMapping{SingleScale} || error( + "Domain `$(domain.name)` uses a multiscale ModelMapping. ", + "The initial SimulationMapping runner only supports single-status domains; ", + "the MTG/domain runner will handle nested multiscale plant domains." + ) + return nothing +end + +_is_single_status_domain(domain::Domain) = _domain_mapping(domain) isa ModelMapping{SingleScale} +_is_graph_domain(domain::Domain) = _domain_mapping(domain) isa ModelMapping{MultiScale} + +function _validate_staged_domain_mapping(domain::Domain) + mapping = _domain_mapping(domain) + mapping isa ModelMapping || error("Domain `$(domain.name)` does not wrap a valid ModelMapping.") + if mapping isa ModelMapping{SingleScale} && !isnothing(domain.selector) + error( + "Domain `$(domain.name)` has a selector but uses a single-status ModelMapping. ", + "Selectors are only supported for MTG-backed, scale-keyed domain mappings in the MTG-domain runner." + ) + end + return nothing +end + +function _collect_matching_nodes(root, predicate) + matches = Any[] + MultiScaleTreeGraph.traverse!(root) do node + predicate(node) && push!(matches, node) + end + return matches +end + +function _is_ancestor_node(ancestor, node) + current = parent(node) + while !isnothing(current) + current === ancestor && return true + current = parent(current) + end + return false +end + +function _validate_domain_graph_roots!(domain::Domain, roots) + for i in eachindex(roots), j in (i + 1):lastindex(roots) + left = roots[i] + right = roots[j] + if _is_ancestor_node(left, right) || _is_ancestor_node(right, left) + error( + "Selector for MTG-backed domain `$(domain.name)` matched overlapping MTG roots ", + "`$(node_id(left))` and `$(node_id(right))`. ", + "Select non-overlapping domain roots, for example plant roots rather than both plants and organs." + ) + end + end + return roots +end + +function _domain_graph_roots(object, domain::Domain) + selector = domain.selector + isnothing(selector) && return Any[object] + selector isa MultiScaleTreeGraph.Node && return Any[selector] + if selector isa Symbol + matches = _collect_matching_nodes(object, node -> symbol(node) == selector) + elseif selector isa Function + matches = _collect_matching_nodes(object, selector) + else + error( + "Unsupported selector for MTG-backed domain `$(domain.name)`: `$(typeof(selector))`. ", + "Use `selector=nothing`, a `MultiScaleTreeGraph.Node`, a scale `Symbol`, or a predicate function." + ) + end + isempty(matches) && error( + "Selector for MTG-backed domain `$(domain.name)` matched $(length(matches)) nodes. ", + "Use a selector that identifies at least one domain root for this MTG-domain runner." + ) + return _validate_domain_graph_roots!(domain, matches) +end + +function _domain_model_specs(domain::Domain) + specs = Dict{DomainModelKey,ModelSpec}() + for (scale, declarations) in _domain_entries(domain) + for (process, spec) in pairs(parse_model_specs(declarations)) + specs[DomainModelKey(domain.name, scale, process)] = spec + end + end + return specs +end + +function _domain_model_clocks(specs::Dict{DomainModelKey,ModelSpec}, timeline::TimelineContext) + clocks = Dict{DomainModelKey,ClockSpec}() + for (key, spec) in specs + clocks[key] = _model_clock(spec, model_(spec), timeline) + end + return clocks +end + +function _domain_dependency_graphs(mapping::SimulationMapping) + graphs = Dict{Symbol,DependencyGraph}() + for domain in mapping.domains + graph = dep(_domain_mapping(domain)) + if !isempty(graph.not_found) + error("Domain `$(domain.name)` has unresolved dependencies: $(graph.not_found).") + end + graphs[domain.name] = graph + end + return graphs +end + +function _domain_for_name(mapping::SimulationMapping, name::Symbol) + for domain in mapping.domains + domain.name == name && return domain + end + error("Unknown domain `$(name)`.") +end + +function _matches(selector::AllDomains, domain::Domain, key::DomainModelKey, spec::ModelSpec; check_var=true) + isnothing(selector.kind) || selector.kind == domain.kind || return false + isnothing(selector.domain) || selector.domain == domain.name || return false + isnothing(selector.scale) || selector.scale == key.scale || return false + isnothing(selector.process) || selector.process == key.process || return false + !check_var || isnothing(selector.var) || selector.var in keys(outputs_(spec)) || return false + return true +end + +function _matches(selector::HardDomains, domain::Domain, key::DomainModelKey, spec::ModelSpec; check_var=false) + isnothing(selector.kind) || selector.kind == domain.kind || return false + isnothing(selector.domain) || selector.domain == domain.name || return false + isnothing(selector.scale) || selector.scale == key.scale || return false + isnothing(selector.process) || selector.process == key.process || return false + return true +end + +function _format_symbol_keys(keys_iter) + syms = sort!(collect(Symbol.(keys_iter)); by=string) + isempty(syms) && return "none" + return join((":" * string(sym) for sym in syms), ", ") +end + +function _domain_candidate_rows( + mapping::SimulationMapping, + specs::Dict{DomainModelKey,ModelSpec}; + selector=nothing, + check_var=true, + max_rows=12, +) + rows = String[] + keys_by_domain = _keys_by_domain(specs) + for domain in mapping.domains + producer_keys = sort!( + copy(get(keys_by_domain, domain.name, DomainModelKey[])); + by=key -> (string(key.scale), string(key.process)), + ) + for producer_key in producer_keys + producer_spec = specs[producer_key] + if selector isa Union{AllDomains,HardDomains} + _matches(selector, domain, producer_key, producer_spec; check_var=check_var) || continue + end + push!( + rows, + "$(producer_key) outputs=($(_format_symbol_keys(keys(outputs_(producer_spec)))))", + ) + end + end + + length(rows) <= max_rows && return join(rows, "\n ") + shown = rows[1:max_rows] + push!(shown, "... and $(length(rows) - max_rows) more") + return join(shown, "\n ") +end + +function _selector_match_error( + mapping::SimulationMapping, + specs::Dict{DomainModelKey,ModelSpec}, + selector::Union{AllDomains,HardDomains}; + context::String, +) + candidates = _domain_candidate_rows(mapping, specs) + message = string( + context, + " did not match any model for selector `", + selector, + "`. Suggested fixes: check `kind`, `domain`, `scale`, and `process`; ", + "if `var` is set, use one of the producer outputs listed below.\n", + "Available producers:\n ", + isempty(candidates) ? "none" : candidates, + ) + if selector isa AllDomains && !isnothing(selector.var) + near_matches = _domain_candidate_rows(mapping, specs; selector=selector, check_var=false) + if !isempty(near_matches) + message = string( + message, + "\nModels matching all selector fields except `var=:", + selector.var, + "`:\n ", + near_matches, + ) + end + end + return message +end + +function _resolve_domain_dependencies(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + bindings = Dict{Tuple{DomainModelKey,Symbol},Vector{DomainModelKey}}() + policies = Dict{Tuple{DomainModelKey,Symbol},SchedulePolicy}() + variables = Dict{Tuple{DomainModelKey,Symbol},Union{Nothing,Symbol}}() + keys_by_domain = _keys_by_domain(specs) + + for (consumer_key, spec) in specs + model_deps = dep(spec) + for (dep_name, selector) in pairs(model_deps) + selector isa AllDomains || continue + resolved = DomainModelKey[] + for domain in mapping.domains + for producer_key in get(keys_by_domain, domain.name, DomainModelKey[]) + producer_key == consumer_key && continue + producer_spec = specs[producer_key] + _matches(selector, domain, producer_key, producer_spec) && push!(resolved, producer_key) + end + end + isempty(resolved) && error( + _selector_match_error( + mapping, + specs, + selector; + context="Domain dependency `$(dep_name)` for consumer `$(consumer_key)`", + ) + ) + binding_key = (consumer_key, dep_name) + bindings[binding_key] = resolved + policies[binding_key] = selector.policy + variables[binding_key] = selector.var + end + end + + return bindings, policies, variables +end + +function _resolve_hard_domain_dependencies(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + bindings = Dict{Tuple{DomainModelKey,Symbol},Vector{DomainModelKey}}() + keys_by_domain = _keys_by_domain(specs) + + for (consumer_key, spec) in specs + model_deps = dep(spec) + for (dep_name, selector) in pairs(model_deps) + selector isa HardDomains || continue + resolved = DomainModelKey[] + for domain in mapping.domains + for producer_key in get(keys_by_domain, domain.name, DomainModelKey[]) + producer_key == consumer_key && continue + producer_spec = specs[producer_key] + _matches(selector, domain, producer_key, producer_spec) && push!(resolved, producer_key) + end + end + isempty(resolved) && error( + _selector_match_error( + mapping, + specs, + selector; + context="Hard domain dependency `$(dep_name)` for consumer `$(consumer_key)`", + ) + ) + bindings[(consumer_key, dep_name)] = resolved + end + end + + return bindings +end + +function _keys_by_domain(specs::Dict{DomainModelKey,ModelSpec}) + keys_by_domain = Dict{Symbol,Vector{DomainModelKey}}() + for key in keys(specs) + push!(get!(keys_by_domain, key.domain, DomainModelKey[]), key) + end + return keys_by_domain +end + +include("route_runtime.jl") +include("environment_bridge.jl") +include("output_publisher.jl") +include("domain_scheduler.jl") +include("graph_domain_runner.jl") +include("domain_run_loops.jl") + +function _build_domain_simulation(mapping::SimulationMapping, meteo; staged_graph_domains=false) + environment = environment_backend(meteo) + _validate_meteo_duration(environment) + timeline = _timeline_context(environment) + foreach(staged_graph_domains ? _validate_staged_domain_mapping : _validate_initial_domain_mapping, mapping.domains) + specs = Dict{DomainModelKey,ModelSpec}() + for domain in mapping.domains + merge!(specs, _domain_model_specs(domain)) + end + mapping = _add_input_routes(mapping, specs) + mapping = _add_route_target_status_defaults(mapping, specs) + + model_clocks = _domain_model_clocks(specs, timeline) + graphs = _domain_dependency_graphs(mapping) + bindings, policies, variables = _resolve_domain_dependencies(mapping, specs) + hard_domain_bindings = _resolve_hard_domain_dependencies(mapping, specs) + route_bindings = _resolve_route_bindings(mapping, specs) + _validate_route_targets(mapping, mapping.routes, specs; staged_graph_domains=staged_graph_domains) + staged_graph_domains && _validate_graph_route_order(mapping, mapping.routes, route_bindings) + route_clocks = _route_clocks(mapping.routes, specs, model_clocks) + validate_meteo_inputs(_domain_model_specs_by_scale(specs), environment) + return DomainSimulation( + mapping, + environment, + specs, + model_clocks, + graphs, + bindings, + variables, + policies, + hard_domain_bindings, + route_bindings, + route_clocks, + Dict{Tuple{DomainModelKey,Symbol},Vector{Pair{Int,Any}}}(), + Dict{Tuple{DomainModelKey,Symbol},Vector{Any}}(), + Dict{Symbol,Any}(domain.name => _domain_mapping(domain) for domain in mapping.domains if _domain_mapping(domain) isa ModelMapping{SingleScale}), + timeline, + ) +end + +function _domain_model_specs_by_scale(specs::Dict{DomainModelKey,ModelSpec}) + by_scale = Dict{Symbol,Dict{Symbol,ModelSpec}}() + for (key, spec) in specs + scale_key = Symbol(string(key.domain), "/", string(key.scale)) + scale_specs = get!(by_scale, scale_key, Dict{Symbol,ModelSpec}()) + scale_specs[key.process] = spec + end + return by_scale +end + +function _resolve_stream_values(simulation::DomainSimulation, producer::DomainModelKey, var::Symbol, steps) + stream = get(simulation.streams, (producer, var), Pair{Int,Any}[]) + vals = Any[] + for (step, value) in stream + step in steps && push!(vals, value) + end + return vals +end + +function _domain_node_values(values::Vector{Any}) + return DomainNodeValues[value for value in values if value isa DomainNodeValues] +end + +function _domain_node_values_have_ids(values::Vector{DomainNodeValues}) + return all(value -> !isnothing(value.ids) && length(value.ids) == length(value.values), values) +end + +function _combine_domain_node_values_by_position(values::Vector{DomainNodeValues}, reducer) + lengths = unique(length(value.values) for value in values) + length(lengths) == 1 || error( + "Cannot aggregate graph-domain values with changing vector lengths because node ids are unavailable. ", + "Publish graph-domain outputs through PlantSimEngine's graph-domain runtime so values can be aligned by node id." + ) + n = only(lengths) + return DomainNodeValues(Any[reducer(Any[value.values[i] for value in values]) for i in 1:n]) +end + +function _combine_domain_node_values_by_id(values::Vector{DomainNodeValues}, reducer) + grouped = Dict{Int,Vector{Any}}() + order = Int[] + for node_values in values + for (id, value) in zip(node_values.ids, node_values.values) + bucket = get!(grouped, id) do + push!(order, id) + Any[] + end + push!(bucket, value) + end + end + return DomainNodeValues(order, Any[reducer(grouped[id]) for id in order]) +end + +function _combine_domain_node_values(values::Vector{Any}, reducer) + node_values = _domain_node_values(values) + _domain_node_values_have_ids(node_values) && return _combine_domain_node_values_by_id(node_values, reducer) + return _combine_domain_node_values_by_position(node_values, reducer) +end + +function _apply_dependency_policy(values::Vector{Any}, policy::SchedulePolicy) + isempty(values) && return nothing + if policy isa HoldLast || policy isa Interpolate + return last(values) + elseif policy isa Integrate + if any(value -> value isa DomainNodeValues, values) + return _combine_domain_node_values(values, sum) + end + return sum(values) + elseif policy isa Aggregate + if any(value -> value isa DomainNodeValues, values) + return _combine_domain_node_values(values, vals -> sum(vals) / length(vals)) + end + return sum(values) / length(values) + end + return last(values) +end + +_public_dependency_value(value) = value isa DomainNodeValues ? value.values : value + +function _push_public_dependency_value!(dest::Vector{Any}, value, flatten::Bool) + isnothing(value) && return dest + if flatten && value isa DomainNodeValues + append!(dest, value.values) + else + push!(dest, _public_dependency_value(value)) + end + return dest +end + +function _window_steps(step::Int, clock::ClockSpec) + start = _window_start_for_clock(clock, float(step)) + return ceil(Int, start):step +end + +""" + dependency_values(ctx, dependency_name, variable) + dependency_values(ctx, dependency_name) + +Return one value per resolved producer for a scene-level `AllDomains` +dependency, applying the selector's temporal policy over the consumer window. +When `AllDomains(...; var=:x)` declares a variable, the two-argument form uses +that variable. +""" +function dependency_values(ctx::DomainRunContext, dependency_name::Symbol, variable=nothing; flatten=false) + sim = ctx.simulation + binding_key = (ctx.consumer, dependency_name) + producers = get(sim.dependency_bindings, binding_key, nothing) + isnothing(producers) && error( + "No resolved dependency named `$(dependency_name)` for `$(ctx.consumer)`. ", + "Declare it with `dep(model) = (; $(dependency_name)=AllDomains(...))`." + ) + declared_var = sim.dependency_variables[binding_key] + resolved_var = isnothing(variable) ? declared_var : Symbol(variable) + isnothing(resolved_var) && error( + "No variable was provided for domain dependency `$(dependency_name)` in `$(ctx.consumer)`. ", + "Call `dependency_values(extra, :$(dependency_name), :variable)` or declare ", + "`AllDomains(...; var=:variable)`." + ) + if !isnothing(declared_var) && !isnothing(variable) && Symbol(variable) != declared_var + error( + "Domain dependency `$(dependency_name)` in `$(ctx.consumer)` was declared with `var=:$(declared_var)`, ", + "but `dependency_values` was called for variable `$(Symbol(variable))`." + ) + end + for producer in producers + producer_spec = sim.model_specs[producer] + resolved_var in keys(outputs_(producer_spec)) || error( + "Domain dependency `$(dependency_name)` resolved producer `$(producer)`, ", + "but that producer does not output variable `$(resolved_var)`." + ) + end + policy = sim.dependency_policies[binding_key] + steps = _window_steps(ctx.step, ctx.clock) + values = Any[] + for producer in producers + value = _apply_dependency_policy(_resolve_stream_values(sim, producer, resolved_var, steps), policy) + _push_public_dependency_value!(values, value, flatten) + end + return values +end + +dependency_values(ctx::DomainRunContext, dependency_name::AbstractString, variable=nothing; flatten=false) = + dependency_values(ctx, Symbol(dependency_name), variable; flatten=flatten) + +function _find_dependency_node(node::SoftDependencyNode, key::DomainModelKey) + node.scale == key.scale && node.process == key.process && return node + for child in node.children + found = _find_dependency_node(child, key) + isnothing(found) || return found + end + return nothing +end + +function _find_dependency_node(graph::DependencyGraph, key::DomainModelKey) + for (_, root) in graph.roots + found = _find_dependency_node(root, key) + isnothing(found) || return found + end + error("No dependency node found for domain model `$(key)`.") +end + +function _single_status_dependency_targets(ctx::DomainRunContext, producer::DomainModelKey) + sim = ctx.simulation + domain = _domain_for_name(sim.mapping, producer.domain) + model_list = _single_scale_model_set_from_mapping(_domain_mapping(domain)) + node = _find_dependency_node(sim.dependency_graphs[producer.domain], producer) + st = status(model_list) + producer_context = DomainRunContext(sim, producer, ctx.step, sim.model_clocks[producer], ctx.constants) + return ModelTarget[ + ModelTarget( + sim, + producer, + node, + ctx.step, + node.value, + model_list.models, + st, + _dependency_target_meteo(sim, producer, st, ctx.step), + ctx.constants, + producer_context, + ), + ] +end + +function _graph_dependency_targets(ctx::DomainRunContext, producer::DomainModelKey) + sim = ctx.simulation + graph_state = sim.domain_states[producer.domain] + targets = ModelTarget[] + for graph_simulation in graph_state.simulations + graph_statuses = status(graph_simulation) + haskey(graph_statuses, producer.scale) || continue + node = _find_dependency_node(dep(graph_simulation), producer) + models_at_scale = get_models(graph_simulation)[producer.scale] + for st in graph_statuses[producer.scale] + push!( + targets, + ModelTarget( + sim, + producer, + node, + ctx.step, + node.value, + models_at_scale, + st, + _dependency_target_meteo(sim, producer, st, ctx.step), + ctx.constants, + graph_simulation, + ), + ) + end + end + return targets +end + +function _dependency_targets_for_producer(ctx::DomainRunContext, producer::DomainModelKey) + state = ctx.simulation.domain_states[producer.domain] + state isa ModelMapping{SingleScale} && return _single_status_dependency_targets(ctx, producer) + state isa DomainGraphState && return _graph_dependency_targets(ctx, producer) + error("Unsupported runtime state for domain `$(producer.domain)`: `$(typeof(state))`.") +end + +""" + dependency_targets(ctx, dependency_name) + +Return executable targets for a resolved `HardDomains` dependency. The parent +model controls when, how often, and in which order targets are executed by +calling [`run_target!`](@ref). +""" +function dependency_targets(ctx::DomainRunContext, dependency_name::Symbol) + sim = ctx.simulation + binding_key = (ctx.consumer, dependency_name) + producers = get(sim.hard_domain_dependency_bindings, binding_key, nothing) + isnothing(producers) && error( + "No hard-domain dependency named `$(dependency_name)` for `$(ctx.consumer)`. ", + "Declare it with `dep(model) = (; $(dependency_name)=HardDomains(...))`." + ) + targets = ModelTarget[] + for producer in producers + append!(targets, _dependency_targets_for_producer(ctx, producer)) + end + return targets +end + +dependency_targets(ctx::DomainRunContext, dependency_name::AbstractString) = + dependency_targets(ctx, Symbol(dependency_name)) + +""" + run_target!(target; meteo=target.meteo, constants=target.constants, extra=target.extra, publish=false) + +Run one executable model target. The call mutates the target's +status, just like a normal hard dependency call. It does not append to domain +streams or outputs unless `publish=true`. +""" +function run_target!( + target::ModelTarget; + meteo=target.meteo, + constants=target.constants, + extra=target.extra, + publish::Bool=false, +) + run!(target.model, target.models, target.status, meteo, constants, extra) + publish && _publish_target!(target) + return target.status +end + +function run_target!( + models, + status, + dependency_name::Symbol; + meteo=nothing, + constants=nothing, + extra=nothing, + publish::Bool=false, +) + target = dependency_target(models, status, dependency_name; meteo=meteo, constants=constants, extra=extra) + return run_target!(target; publish=publish) +end + +run_target!(models, status, dependency_name::AbstractString; kwargs...) = + run_target!(models, status, Symbol(dependency_name); kwargs...) + +""" + run_call!(target; kwargs...) + run_call!(models, status, dependency_name; kwargs...) + +Unified scene/object spelling for manually executing a model call handle. +This currently delegates to `run_target!` while `ModelTarget` remains the +runtime carrier for hard-domain calls. +""" +run_call!(args...; kwargs...) = run_target!(args...; kwargs...) + +function _domain_context_for(simulation::DomainSimulation, domain::Domain, node::SoftDependencyNode, step::Int, constants=nothing) + key = DomainModelKey(domain.name, node.scale, node.process) + return DomainRunContext(simulation, key, step, simulation.model_clocks[key], constants) +end + +function _run_domain_node!( + simulation::DomainSimulation, + domain::Domain, + node::SoftDependencyNode, + model_list::SingleScaleModelSet, + constants, + step::Int, + ran::Set{DomainModelKey}; + phase::Symbol=:normal, +) + key = DomainModelKey(domain.name, node.scale, node.process) + if _domain_node_due(simulation, domain, node, step) && + _domain_parents_ready(simulation, domain, node, step, ran) && + _should_visit_domain_node(simulation, domain, node; phase=phase) && + !(key in ran) + ctx = _domain_context_for(simulation, domain, node, step, constants) + model_spec = simulation.model_specs[key] + meteo_for_model = _domain_environment_for_model( + simulation, + domain, + node, + model_spec, + status(model_list), + step, + ) + run!(node.value, model_list.models, status(model_list), meteo_for_model, constants, ctx) + _scatter_domain_environment_outputs!(simulation, domain, node, model_spec, status(model_list), step) + push!(ran, key) + _publish_domain_model_outputs!(simulation, domain, node, status(model_list), step) + for hard_child in node.hard_dependency + _scatter_domain_hard_dependency_environment_outputs!(simulation, domain, hard_child, status(model_list), step) + _publish_domain_hard_dependency_outputs!(simulation, domain, hard_child, status(model_list), step) + end + end + + for child in node.children + _run_domain_node!(simulation, domain, child, model_list, constants, step, ran; phase=phase) + end + return nothing +end + +function _run_domain_models!( + simulation::DomainSimulation, + domain::Domain, + constants, + step::Int; + phase::Symbol=:normal, +) + mapping = _domain_mapping(domain) + model_list = _single_scale_model_set_from_mapping(mapping) + ran = Set{DomainModelKey}() + for (_, root) in simulation.dependency_graphs[domain.name].roots + _run_domain_node!(simulation, domain, root, model_list, constants, step, ran; phase=phase) + end + return ran +end + +""" + run!(mapping::SimulationMapping, meteo=nothing, constants=PlantMeteo.Constants(); check=true) + +Run the initial domain-aware simulation path. This runner is deliberately +limited to single-status domains, but it schedules each model with its own +effective timestep. It is intended to make the multi-domain API executable +while the full MTG path is implemented. +""" +function run!( + mapping::SimulationMapping, + meteo=nothing, + constants=PlantMeteo.Constants(); + check=true +) + simulation = _build_domain_simulation(mapping, meteo) + nsteps = get_nsteps(simulation.environment) + return _run_single_status_domain_simulation!(simulation, constants, nsteps) +end + +""" + run!(mtg, mapping::SimulationMapping, meteo=nothing, constants=PlantMeteo.Constants(); ...) + +Run a multi-domain simulation where MTG-backed domains are selected from `mtg` +and executed with the existing `GraphSimulation` engine. + +The runner advances all domains one base timestep at a time. Domains whose +`kind` is not `:scene` run first in mapping order, then `:scene` domains run so +they can consume plant, soil, and graph-domain streams from the same timestep. +Routes into graph domains are supported for `OneToManyBroadcast()` when the +source domain runs earlier in the timestep. +""" +function run!( + object::MultiScaleTreeGraph.Node, + mapping::SimulationMapping, + meteo=nothing, + constants=PlantMeteo.Constants(); + nsteps=nothing, + check=true, + executor=SequentialEx(), + type_promotion=nothing, +) + simulation = _build_domain_simulation(mapping, meteo; staged_graph_domains=true) + raw_meteo = _raw_meteo_for_staged_graph_domains(simulation.environment) + isnothing(nsteps) && (nsteps = get_nsteps(simulation.environment)) + return _run_staged_graph_domain_simulation!( + simulation, + object, + raw_meteo, + constants, + nsteps; + check=check, + executor=executor, + type_promotion=type_promotion, + ) +end + +""" + explain_domains(mapping_or_simulation) + +Return structured rows describing domains in a simulation mapping. +""" +function explain_domains(mapping::SimulationMapping) + return [ + (domain=domain.name, kind=domain.kind, mapping=typeof(domain.mapping), selector=domain.selector) + for domain in mapping.domains + ] +end + +explain_domains(simulation::DomainSimulation) = explain_domains(simulation.mapping) + +function _domain_model_rows(mapping::SimulationMapping) + rows = NamedTuple[] + for domain in mapping.domains + for (scale, declarations) in _domain_entries(domain) + for (process, spec) in pairs(parse_model_specs(declarations)) + key = DomainModelKey(domain.name, scale, process) + push!(rows, ( + key=key, + domain=domain.name, + kind=domain.kind, + scale=scale, + process=process, + model=typeof(model_(spec)), + timestep=timestep(spec), + inputs=inputs_(spec), + outputs=outputs_(spec), + meteo_inputs=meteo_inputs_(spec), + meteo_outputs=meteo_outputs_(spec), + updates=updates(spec), + )) + end + end + end + return rows +end + +""" + explain_domain_models(mapping_or_simulation) + +Return structured rows for every model process inside every domain. +""" +explain_domain_models(mapping::SimulationMapping) = _domain_model_rows(mapping) +explain_domain_models(simulation::DomainSimulation) = explain_domain_models(simulation.mapping) + +""" + explain_domain_statuses(simulation) + +Return structured rows describing runtime status counts by domain and scale. +For graph domains, one row is returned per scale. For single-status domains, +the scale is `:Default`. +""" +function explain_domain_statuses(simulation::DomainSimulation) + rows = NamedTuple[] + for domain in simulation.mapping.domains + state = get(simulation.domain_states, domain.name, nothing) + if state isa DomainGraphState + graph_statuses = status(state) + declared_scales = Symbol[first(entry) for entry in _domain_entries(domain)] + runtime_scales = collect(keys(graph_statuses)) + for scale in sort!(unique!(vcat(declared_scales, runtime_scales)); by=string) + statuses_at_scale = get(graph_statuses, scale, Status[]) + push!(rows, ( + domain=domain.name, + kind=domain.kind, + scale=scale, + nstatuses=length(statuses_at_scale), + state=typeof(state), + )) + end + elseif state isa ModelMapping{SingleScale} + push!(rows, ( + domain=domain.name, + kind=domain.kind, + scale=:Default, + nstatuses=1, + state=typeof(state), + )) + end + end + return rows +end + +""" + explain_schedule(simulation) + +Return structured rows describing effective per-domain schedules. +""" +function explain_schedule(simulation::DomainSimulation) + rows = NamedTuple[] + for domain in simulation.mapping.domains, (key, clock) in simulation.model_clocks + key.domain == domain.name || continue + push!(rows, ( + domain=domain.name, + kind=domain.kind, + scale=key.scale, + process=key.process, + dt_steps=clock.dt, + phase=clock.phase, + dt_seconds=clock.dt * simulation.timeline.base_step_seconds, + )) + end + return rows +end + +""" + explain_domain_dependencies(simulation) + +Return structured rows describing resolved cross-domain dependencies. +""" +function explain_domain_dependencies(simulation::DomainSimulation) + rows = NamedTuple[] + for ((consumer, name), producers) in simulation.dependency_bindings + policy = simulation.dependency_policies[(consumer, name)] + variable = simulation.dependency_variables[(consumer, name)] + for producer in producers + push!(rows, ( + mode=:value, + consumer=consumer, + dependency=name, + producer=producer, + variable=variable, + policy=typeof(policy), + )) + end + end + for ((consumer, name), producers) in simulation.hard_domain_dependency_bindings + for producer in producers + push!(rows, ( + mode=:hard_domain, + consumer=consumer, + dependency=name, + producer=producer, + variable=nothing, + policy=nothing, + )) + end + end + return rows +end + +""" + explain_routes(simulation) + +Return structured rows describing resolved explicit cross-domain routes. +""" +function explain_routes(simulation::DomainSimulation) + rows = NamedTuple[] + for (i, route) in enumerate(simulation.mapping.routes) + clock = simulation.route_clocks[i] + for producer in simulation.route_bindings[i] + push!(rows, ( + route=i, + from=route.from, + to=route.to, + producer=producer, + source_var=route.from.var, + target_var=route.to.var, + cardinality=typeof(route.cardinality), + policy=typeof(route.policy), + dt_steps=clock.dt, + phase=clock.phase, + dt_seconds=clock.dt * simulation.timeline.base_step_seconds, + )) + end + end + return rows +end diff --git a/src/domains/environment_bridge.jl b/src/domains/environment_bridge.jl new file mode 100644 index 000000000..d68d9baef --- /dev/null +++ b/src/domains/environment_bridge.jl @@ -0,0 +1,171 @@ +function _domain_environment_entities(simulation::DomainSimulation, domain::Domain) + state = get(simulation.domain_states, domain.name, nothing) + entities = NamedTuple[] + if state isa DomainGraphState + for (scale, statuses_at_scale) in status(state) + push!(entities, ( + domain=domain.name, + kind=domain.kind, + scale=scale, + statuses=statuses_at_scale, + state=state, + )) + end + elseif state isa ModelMapping{SingleScale} + push!(entities, ( + domain=domain.name, + kind=domain.kind, + scale=:Default, + statuses=Status[status(state)], + state=state, + )) + end + return entities +end + +function _update_domain_environment_index!(simulation::DomainSimulation, domain::Domain) + return update_index!(simulation.environment, _domain_environment_entities(simulation, domain)) +end + +_domain_environment_support(domain::Domain, node::AbstractDependencyNode, status) = + EnvironmentSupport(domain.name, node.scale, node.process, status) + +_domain_environment_support(key::DomainModelKey, status) = + EnvironmentSupport(key.domain, key.scale, key.process, status) + +function _sample_domain_environment_at_time(simulation::DomainSimulation, support::EnvironmentSupport, t, model_spec::ModelSpec) + return sample_environment(simulation.environment, support, t, model_spec) +end + +function _sample_domain_environment_at_step(simulation::DomainSimulation, support::EnvironmentSupport, step::Int, model_spec::ModelSpec) + t = _time_from_step(step, simulation.timeline) + return _sample_domain_environment_at_time(simulation, support, t, model_spec) +end + +function _dependency_target_meteo(simulation::DomainSimulation, key::DomainModelKey, st, step::Int) + spec = simulation.model_specs[key] + support = _domain_environment_support(key, st) + return _sample_domain_environment_at_step(simulation, support, step, spec) +end + +function _domain_environment_for_model( + simulation::DomainSimulation, + domain::Domain, + node::SoftDependencyNode, + model_spec::ModelSpec, + status, + step::Int +) + support = _domain_environment_support(domain, node, status) + return _sample_domain_environment_at_step(simulation, support, step, model_spec) +end + +function _scatter_domain_environment_outputs_at_time!( + simulation::DomainSimulation, + domain::Domain, + node::AbstractDependencyNode, + model_spec::ModelSpec, + status, + t +) + isempty(keys(meteo_outputs_(model_spec))) && return nothing + support = _domain_environment_support(domain, node, status) + return scatter_environment_outputs!(simulation.environment, support, t, model_spec, status) +end + +function _scatter_domain_environment_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::AbstractDependencyNode, + model_spec::ModelSpec, + status, + step::Int +) + t = _time_from_step(step, simulation.timeline) + return _scatter_domain_environment_outputs_at_time!(simulation, domain, node, model_spec, status, t) +end + +function _scatter_domain_hard_dependency_environment_outputs_at_time!( + simulation::DomainSimulation, + domain::Domain, + node::HardDependencyNode, + status, + t +) + key = DomainModelKey(domain.name, node.scale, node.process) + if haskey(simulation.model_specs, key) + spec = simulation.model_specs[key] + _scatter_domain_environment_outputs_at_time!(simulation, domain, node, spec, status, t) + end + for child in node.children + _scatter_domain_hard_dependency_environment_outputs_at_time!(simulation, domain, child, status, t) + end + return nothing +end + +function _scatter_domain_hard_dependency_environment_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::HardDependencyNode, + status, + step::Int +) + t = _time_from_step(step, simulation.timeline) + return _scatter_domain_hard_dependency_environment_outputs_at_time!(simulation, domain, node, status, t) +end + +function _raw_meteo_for_staged_graph_domains(environment::GlobalConstant) + return environment_meteo(environment) +end + +function _raw_meteo_for_staged_graph_domains(environment::AbstractEnvironmentBackend) + return environment +end + +function _graph_domain_environment_for_model( + simulation::DomainSimulation, + domain::Domain, + node::SoftDependencyNode, + status, + t, + model_clock::ClockSpec, + model_spec::ModelSpec, + meteo, + meteo_sampler, + multirate::Bool +) + if simulation.environment isa GlobalConstant + return multirate ? _sample_meteo_for_model(meteo_sampler, meteo, round(Int, t), model_clock, model_spec) : meteo + end + support = _domain_environment_support(domain, node, status) + return _sample_domain_environment_at_time(simulation, support, t, model_spec) +end + +function _meteo_for_graph_step(meteo, step::Int, nsteps::Int) + return _meteo_row_at_step(meteo, step) +end + +function _meteo_for_graph_step(backend::AbstractEnvironmentBackend, step::Int, nsteps::Int) + return backend +end + +function _scatter_graph_domain_environment_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::AbstractDependencyNode, + model_spec::ModelSpec, + status, + t +) + return _scatter_domain_environment_outputs_at_time!(simulation, domain, node, model_spec, status, t) +end + +function _scatter_graph_domain_hard_dependency_environment_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::HardDependencyNode, + status, + t +) + return _scatter_domain_hard_dependency_environment_outputs_at_time!(simulation, domain, node, status, t) +end diff --git a/src/domains/graph_domain_runner.jl b/src/domains/graph_domain_runner.jl new file mode 100644 index 000000000..a9955ef5c --- /dev/null +++ b/src/domains/graph_domain_runner.jl @@ -0,0 +1,135 @@ +function _prepare_graph_domain_runtime!( + simulation::DomainSimulation, + domain::Domain, + object, + meteo, + constants, + nsteps::Int; + check=true, + executor=SequentialEx(), + type_promotion=_type_promotion(_domain_mapping(domain)), +) + roots = _domain_graph_roots(object, domain) + graph_simulations = GraphSimulation[] + for root in roots + _materialize_graph_route_attributes_for_domain!(simulation, domain, root, 1) + push!( + graph_simulations, + GraphSimulation( + root, + _domain_mapping(domain); + nsteps=nsteps, + check=check, + outputs=nothing, + type_promotion=type_promotion, + ), + ) + end + graph_state = DomainGraphState(graph_simulations) + simulation.domain_states[domain.name] = graph_state + representative_simulation = first(graph_simulations) + effective_multirate = _effective_multirate(representative_simulation) + dep_graph = dep(representative_simulation) + timeline = _timeline_context(meteo) + meteo_sampler = effective_multirate ? _prepare_meteo_sampler(meteo) : nothing + runtime_clock_rows = _runtime_clock_rows(representative_simulation, timeline, dep_graph) + effective_executor = executor + validate_meteo_inputs(get_model_specs(representative_simulation), meteo) + _validate_meteo_derived_timestep_requirements!(runtime_clock_rows, timeline) + if effective_multirate + if executor != SequentialEx() + @warn string( + "Multi-rate MTG domain `$(domain.name)` currently executes sequentially. ", + "Provided `executor=$(executor)` is ignored in this mode. ", + "Use `executor=SequentialEx()` to silence this warning." + ) maxlog = 1 + effective_executor = SequentialEx() + end + _warn_if_no_model_runs_at_base_timestep(runtime_clock_rows, timeline) + for graph_simulation in graph_simulations + validate_canonical_publishers(graph_simulation) + configure_temporal_buffers!(graph_simulation, timeline) + end + end + return DomainGraphRuntime(graph_state, meteo, constants, effective_multirate, timeline, meteo_sampler, effective_executor) +end + +function _run_graph_domain_step!( + simulation::DomainSimulation, + domain::Domain, + runtime::DomainGraphRuntime, + step::Int, + nsteps::Int; + check=true, + phase::Symbol=:normal, +) + meteo_i = _meteo_for_graph_step(runtime.meteo, step, nsteps) + meteo_provider = (node, status, i, t, model_clock, model_spec, meteo, meteo_sampler, multirate) -> + _graph_domain_environment_for_model( + simulation, + domain, + node, + status, + t, + model_clock, + model_spec, + meteo, + meteo_sampler, + multirate, + ) + after_model_run = (node, model_spec, status, i, t) -> begin + _scatter_graph_domain_environment_outputs!(simulation, domain, node, model_spec, status, t) + for hard_child in node.hard_dependency + _scatter_graph_domain_hard_dependency_environment_outputs!(simulation, domain, hard_child, status, t) + end + nothing + end + skip_model_run = node -> !_should_visit_domain_node(simulation, domain, node; phase=phase) + for graph_simulation in runtime.state.simulations + _materialize_graph_routes_for_domain!(simulation, domain, graph_simulation, step) + roots = collect(dep(graph_simulation).roots) + models = get_models(graph_simulation) + for (_, dependency_node) in roots + run_node_multiscale!( + graph_simulation, + dependency_node, + step, + models, + meteo_i, + runtime.constants, + graph_simulation, + check, + runtime.executor, + runtime.effective_multirate, + runtime.timeline, + runtime.meteo_sampler; + meteo_provider=meteo_provider, + after_model_run=after_model_run, + skip_model_run=skip_model_run, + ) + end + if phase == :normal + runtime.effective_multirate && update_requested_outputs!(graph_simulation, _time_from_step(step, runtime.timeline)) + save_results!(graph_simulation, step) + end + end + _publish_graph_domain_step_outputs!( + simulation, + domain, + runtime.state, + step; + effective_multirate=runtime.effective_multirate, + phase=phase, + ) + _update_domain_environment_index!(simulation, domain) + return runtime.state +end + +function _finalize_graph_domain_runtime!(runtime::DomainGraphRuntime) + for graph_simulation in runtime.state.simulations + for (organ, index) in graph_simulation.outputs_index + resize!(outputs(graph_simulation)[organ], index - 1) + end + end + return runtime.state +end diff --git a/src/domains/output_publisher.jl b/src/domains/output_publisher.jl new file mode 100644 index 000000000..51b5feb2b --- /dev/null +++ b/src/domains/output_publisher.jl @@ -0,0 +1,104 @@ +function _publish_domain_model_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::AbstractDependencyNode, + status, + step::Int +) + key = DomainModelKey(domain.name, node.scale, node.process) + haskey(simulation.model_specs, key) || return nothing + spec = simulation.model_specs[key] + for out_var in keys(outputs_(spec)) + stream_key = (key, out_var) + value = status[out_var] + push!(get!(simulation.streams, stream_key, Pair{Int,Any}[]), step => value) + push!(get!(simulation.outputs, stream_key, Any[]), value) + end + return nothing +end + +function _publish_domain_hard_dependency_outputs!( + simulation::DomainSimulation, + domain::Domain, + node::HardDependencyNode, + status, + step::Int +) + _publish_domain_model_outputs!(simulation, domain, node, status, step) + for child in node.children + _publish_domain_hard_dependency_outputs!(simulation, domain, child, status, step) + end + return nothing +end + +function _publish_graph_domain_step_outputs!( + simulation::DomainSimulation, + domain::Domain, + graph_state::DomainGraphState, + step::Int; + effective_multirate::Bool=false, + phase::Symbol=:normal, +) + graph_statuses = status(graph_state) + for (key, spec) in simulation.model_specs + key.domain == domain.name || continue + _should_publish_domain_key(simulation, domain, key; phase=phase) || continue + if effective_multirate + clock = simulation.model_clocks[key] + _should_run_at_time(clock, float(step)) || continue + end + haskey(graph_statuses, key.scale) || continue + for out_var in keys(outputs_(spec)) + stream_key = (key, out_var) + ids = Int[] + values = Any[] + for st in graph_statuses[key.scale] + out_var in propertynames(st) || continue + push!(ids, node_id(st.node)) + push!(values, st[out_var]) + end + push!(get!(simulation.streams, stream_key, Pair{Int,Any}[]), step => DomainNodeValues(ids, values)) + push!(get!(simulation.outputs, stream_key, Any[]), values) + end + end + return nothing +end + +function _publish_graph_target!(target::ModelTarget) + spec = target.simulation.model_specs[target.key] + domain = _domain_for_name(target.simulation.mapping, target.key.domain) + t = _time_from_step(target.step, target.simulation.timeline) + _scatter_graph_domain_environment_outputs!(target.simulation, domain, target.node, spec, target.status, t) + for hard_child in target.node.hard_dependency + _scatter_graph_domain_hard_dependency_environment_outputs!(target.simulation, domain, hard_child, target.status, t) + end + for out_var in keys(outputs_(spec)) + out_var in propertynames(target.status) || continue + stream_key = (target.key, out_var) + value = target.status[out_var] + push!(get!(target.simulation.streams, stream_key, Pair{Int,Any}[]), target.step => value) + push!(get!(target.simulation.outputs, stream_key, Any[]), value) + end + for hard_child in target.node.hard_dependency + _publish_domain_hard_dependency_outputs!(target.simulation, domain, hard_child, target.status, target.step) + end + return nothing +end + +function _publish_target!(target::ModelTarget) + isnothing(target.simulation) && error( + "`publish=true` requires a target created by `dependency_targets(extra, name)` in a domain simulation." + ) + target.extra isa GraphSimulation && return _publish_graph_target!(target) + domain = _domain_for_name(target.simulation.mapping, target.key.domain) + spec = target.simulation.model_specs[target.key] + _scatter_domain_environment_outputs!(target.simulation, domain, target.node, spec, target.status, target.step) + for hard_child in target.node.hard_dependency + _scatter_domain_hard_dependency_environment_outputs!(target.simulation, domain, hard_child, target.status, target.step) + end + _publish_domain_model_outputs!(target.simulation, domain, target.node, target.status, target.step) + for hard_child in target.node.hard_dependency + _publish_domain_hard_dependency_outputs!(target.simulation, domain, hard_child, target.status, target.step) + end + return nothing +end diff --git a/src/domains/route_runtime.jl b/src/domains/route_runtime.jl new file mode 100644 index 000000000..eb8091936 --- /dev/null +++ b/src/domains/route_runtime.jl @@ -0,0 +1,386 @@ +function _resolve_selector_matches( + mapping::SimulationMapping, + specs::Dict{DomainModelKey,ModelSpec}, + selector::Union{AllDomains,HardDomains}; + context::String, +) + resolved = DomainModelKey[] + keys_by_domain = _keys_by_domain(specs) + for domain in mapping.domains + for producer_key in get(keys_by_domain, domain.name, DomainModelKey[]) + producer_spec = specs[producer_key] + _matches(selector, domain, producer_key, producer_spec) && push!(resolved, producer_key) + end + end + isempty(resolved) && error( + _selector_match_error(mapping, specs, selector; context=context) + ) + return resolved +end + +function _resolve_route_bindings(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + bindings = Vector{DomainModelKey}[] + for (i, route) in enumerate(mapping.routes) + push!( + bindings, + _resolve_selector_matches( + mapping, + specs, + route.from; + context="Route $(i) from `$(route.from)`", + ), + ) + end + return bindings +end + +function _has_route_to_input(routes::Vector{Route}, consumer_key::DomainModelKey, input_var::Symbol) + any(routes) do route + route.to.domain == consumer_key.domain && + route.to.scale == consumer_key.scale && + route.to.var == input_var && + (isnothing(route.to.process) || route.to.process == consumer_key.process) + end +end + +function _all_domains_from_input_selector(selector::AbstractObjectMultiplicity, input_var::Symbol) + c = criteria(selector) + unsupported = (:name, :relation, :within, :window) + any(key -> haskey(c, key) && !isnothing(getproperty(c, key)), unsupported) && return nothing + + source_var = haskey(c, :var) ? c.var : input_var + policy = haskey(c, :policy) ? _as_schedule_policy(c.policy; context="Inputs route policy for `$(input_var)`") : HoldLast() + return AllDomains( + kind=haskey(c, :kind) ? c.kind : nothing, + domain=haskey(c, :domain) ? c.domain : nothing, + scale=haskey(c, :scale) ? c.scale : nothing, + process=haskey(c, :process) ? c.process : nothing, + var=source_var, + policy=policy, + ) +end + +function _single_value_route_reducer(values) + length(values) == 1 || error( + "`One(...)` input route expected exactly one resolved value, got $(length(values))." + ) + return only(values) +end + +function _route_cardinality_from_input_selector(selector::AbstractObjectMultiplicity) + selector isa Many && return ManyToOneVector() + selector isa Union{One,OptionalOne} && return ManyToOneAggregate(_single_value_route_reducer) + return nothing +end + +function _routes_from_value_inputs(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + routes = Route[] + existing_routes = copy(mapping.routes) + for (consumer_key, spec) in specs + bindings = value_inputs(spec) + bindings isa NamedTuple || continue + for (input_var, selector) in pairs(bindings) + input_sym = Symbol(input_var) + input_sym in keys(inputs_(spec)) || continue + selector isa AbstractObjectMultiplicity || continue + _has_route_to_input(existing_routes, consumer_key, input_sym) && continue + + source = _all_domains_from_input_selector(selector, input_sym) + isnothing(source) && continue + cardinality = _route_cardinality_from_input_selector(selector) + isnothing(cardinality) && continue + + target = DomainRouteTarget( + consumer_key.domain; + scale=consumer_key.scale, + var=input_sym, + process=consumer_key.process, + ) + route = Route(from=source, to=target, cardinality=cardinality) + push!(routes, route) + push!(existing_routes, route) + end + end + return routes +end + +function _add_input_routes(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + generated_routes = _routes_from_value_inputs(mapping, specs) + isempty(generated_routes) && return mapping + return SimulationMapping(mapping.domains...; routes=(mapping.routes..., generated_routes...)) +end + +function _route_target_input_default(route::Route, specs::Dict{DomainModelKey,ModelSpec}) + consumer_key = _route_target_consumer_key(route, specs) + isnothing(consumer_key) && return nothing + consumer_inputs = inputs_(specs[consumer_key]) + target_var = route.to.var + target_var in keys(consumer_inputs) || return nothing + return getproperty(consumer_inputs, target_var) +end + +function _route_target_status_defaults(mapping::SimulationMapping, domain::Domain, specs::Dict{DomainModelKey,ModelSpec}) + defaults = NamedTuple() + target_mapping = _domain_mapping(domain) + target_mapping isa ModelMapping{SingleScale} || return defaults + target_status = status(target_mapping) + + for route in mapping.routes + target = route.to + target.domain == domain.name || continue + target.scale == :Default || continue + target.var in propertynames(target_status) && continue + target.var in keys(defaults) && continue + + default = _route_target_input_default(route, specs) + isnothing(default) && continue + defaults = merge(defaults, NamedTuple{(target.var,)}((default,))) + end + + return defaults +end + +function _add_route_target_status_defaults(mapping::SimulationMapping, specs::Dict{DomainModelKey,ModelSpec}) + domains = Domain[] + changed = false + + for domain in mapping.domains + target_mapping = _domain_mapping(domain) + defaults = _route_target_status_defaults(mapping, domain, specs) + if target_mapping isa ModelMapping{SingleScale} && !isempty(keys(defaults)) + augmented_status = Status(merge(defaults, NamedTuple(status(target_mapping)))) + target_mapping = copy(target_mapping, augmented_status) + changed = true + end + push!(domains, Domain(domain.name, domain.kind, target_mapping, domain.selector)) + end + + changed || return mapping + return SimulationMapping(domains...; routes=mapping.routes) +end + +function _route_target_consumer_key(route::Route, specs::Dict{DomainModelKey,ModelSpec}) + target = route.to + if !isnothing(target.process) + key = DomainModelKey(target.domain, target.scale, target.process) + haskey(specs, key) || error( + "Route target process `$(key)` does not exist." + ) + target.var in keys(inputs_(specs[key])) || error( + "Route target process `$(key)` does not consume variable `$(target.var)`. ", + "Use a process that declares `$(target.var)` in `inputs_`, or omit `process=...` ", + "if the route should only materialize a domain status variable." + ) + return key + end + + consumers = DomainModelKey[] + for (key, spec) in specs + key.domain == target.domain || continue + key.scale == target.scale || continue + target.var in keys(inputs_(spec)) || continue + push!(consumers, key) + end + + isempty(consumers) && return nothing + length(consumers) == 1 || error( + "Route target `$(target.domain)/$(target.scale)/$(target.var)` is consumed by several models: ", + join(consumers, ", "), + ". Specify `process=...` in `DomainRouteTarget` so the route clock is unambiguous." + ) + return only(consumers) +end + +function _validate_route_targets(mapping::SimulationMapping, routes::Vector{Route}, specs::Dict{DomainModelKey,ModelSpec}; staged_graph_domains=false) + for (i, route) in enumerate(routes) + target = route.to + domain = _domain_for_name(mapping, target.domain) + target_mapping = _domain_mapping(domain) + if target_mapping isa ModelMapping{MultiScale} + staged_graph_domains || error( + "Route $(i) targets MTG-backed domain `$(target.domain)`, but this runner only supports single-status domains." + ) + route.cardinality isa OneToManyBroadcast || error( + "Route $(i) targets MTG-backed domain `$(target.domain)`. ", + "The MTG-domain runner only supports `OneToManyBroadcast()` routes into graph domains." + ) + target.scale == :Default && error( + "Route $(i) targets MTG-backed domain `$(target.domain)` but uses `scale=:Default`. ", + "Specify the target graph scale, for example `scale=:Leaf`." + ) + isnothing(_route_target_consumer_key(route, specs)) && error( + "Route $(i) targets graph variable `$(target.var)` in `$(target.domain)/$(target.scale)`, ", + "but no target process consumes that variable. Specify `process=...` or add the variable to one model's `inputs_`." + ) + continue + end + target.scale == :Default || error( + "Route $(i) target `$(target.domain)/$(target.scale)/$(target.var)` is not supported by the single-status domain runner. ", + "Use `scale=:Default` for single-status targets, or target an MTG-backed domain with a supported graph route cardinality." + ) + st = status(target_mapping) + target.var in propertynames(st) || error( + "Route $(i) target status `$(target.domain)/$(target.scale)` does not contain variable `$(target.var)`. ", + "Initialize it in the target domain status so the route can materialize its value." + ) + _route_target_consumer_key(route, specs) + end + return nothing +end + +function _validate_graph_route_order( + mapping::SimulationMapping, + routes::Vector{Route}, + route_bindings::Vector{Vector{DomainModelKey}}, +) + _domain_run_order(mapping, route_bindings) + return nothing +end + +function _route_clocks(routes::Vector{Route}, specs::Dict{DomainModelKey,ModelSpec}, model_clocks) + clocks = ClockSpec[] + for route in routes + consumer_key = _route_target_consumer_key(route, specs) + if isnothing(consumer_key) + push!(clocks, ClockSpec(1.0, 0.0)) + else + push!(clocks, model_clocks[consumer_key]) + end + end + return clocks +end + +function _route_due(simulation::DomainSimulation, route_index::Int, step::Int) + clock = simulation.route_clocks[route_index] + return _should_run_at_time(clock, float(step)) +end + +function _route_producer_values(simulation::DomainSimulation, route_index::Int, step::Int) + route = simulation.mapping.routes[route_index] + source_var = route.from.var + steps = _window_steps(step, simulation.route_clocks[route_index]) + return Any[ + _apply_dependency_policy(_resolve_stream_values(simulation, producer, source_var, steps), route.policy) + for producer in simulation.route_bindings[route_index] + ] +end + +function _route_value_items(values::Vector{Any}) + items = Any[] + for value in values + isnothing(value) && continue + if value isa DomainNodeValues + append!(items, value.values) + else + push!(items, value) + end + end + return items +end + +function _materialize_route_value(values::Vector{Any}, cardinality::ManyToOneVector) + return _route_value_items(values) +end + +function _materialize_route_value(values::Vector{Any}, cardinality::ManyToOneAggregate) + return cardinality.reducer(_route_value_items(values)) +end + +function _materialize_graph_broadcast_value(values::Vector{Any}, route_index::Int) + items = _route_value_items(values) + length(items) == 1 || error( + "Route $(route_index) uses `OneToManyBroadcast()` into an MTG-backed domain and resolved $(length(items)) values. ", + "Use a selector that resolves one source value, or aggregate upstream before broadcasting." + ) + return only(items) +end + +function _materialize_route_value(values::Vector{Any}, cardinality::RouteCardinality) + error( + "Route cardinality `$(typeof(cardinality))` is declared but not implemented in the single-status domain runner. ", + "Use `ManyToOneVector()` or `ManyToOneAggregate(...)` for single-status targets. ", + "`OneToManyBroadcast()` is supported for MTG-backed target domains." + ) +end + +function _set_route_target_value!(simulation::DomainSimulation, route::Route, value) + target = route.to + domain = _domain_for_name(simulation.mapping, target.domain) + target.scale == :Default || error( + "Route target `$(target.domain)/$(target.scale)/$(target.var)` is not supported by the single-status domain runner. ", + "Use `scale=:Default` for single-status targets, or target an MTG-backed domain with a supported graph route cardinality." + ) + st = status(simulation, domain.name) + target.var in propertynames(st) || error( + "Route target status `$(target.domain)/$(target.scale)` does not contain variable `$(target.var)`. ", + "Initialize it in the target domain status so the route can materialize its value." + ) + st[target.var] = value + return nothing +end + +function _materialize_routes_for_domain!(simulation::DomainSimulation, domain::Domain, step::Int) + for (i, route) in enumerate(simulation.mapping.routes) + route.to.domain == domain.name || continue + _route_due(simulation, i, step) || continue + values = _route_producer_values(simulation, i, step) + value = _materialize_route_value(values, route.cardinality) + _set_route_target_value!(simulation, route, value) + end + return nothing +end + +function _materialize_graph_routes_for_domain!( + simulation::DomainSimulation, + domain::Domain, + graph_simulation::GraphSimulation, + step::Int, +) + for (i, route) in enumerate(simulation.mapping.routes) + route.to.domain == domain.name || continue + route.cardinality isa OneToManyBroadcast || continue + _route_due(simulation, i, step) || continue + target = route.to + graph_statuses = status(graph_simulation) + haskey(graph_statuses, target.scale) || error( + "Route $(i) targets `$(target.domain)/$(target.scale)/$(target.var)`, ", + "but the selected graph domain has no statuses at scale `$(target.scale)`." + ) + values = _route_producer_values(simulation, i, step) + value = _materialize_graph_broadcast_value(values, i) + for st in graph_statuses[target.scale] + target.var in propertynames(st) || error( + "Route $(i) targets `$(target.domain)/$(target.scale)/$(target.var)`, ", + "but one target status does not contain variable `$(target.var)`." + ) + st[target.var] = value + end + end + return nothing +end + +function _materialize_graph_route_attributes_for_domain!( + simulation::DomainSimulation, + domain::Domain, + root, + step::Int, +) + for (i, route) in enumerate(simulation.mapping.routes) + route.to.domain == domain.name || continue + route.cardinality isa OneToManyBroadcast || continue + target = route.to + values = _route_producer_values(simulation, i, step) + value = _materialize_graph_broadcast_value(values, i) + matched = 0 + MultiScaleTreeGraph.traverse!(root) do node + symbol(node) == target.scale || return + node[target.var] = value + matched += 1 + end + matched > 0 || error( + "Route $(i) targets `$(target.domain)/$(target.scale)/$(target.var)`, ", + "but the selected graph domain has no nodes at scale `$(target.scale)`." + ) + end + return nothing +end diff --git a/src/domains/routes.jl b/src/domains/routes.jl new file mode 100644 index 000000000..ad4da48d1 --- /dev/null +++ b/src/domains/routes.jl @@ -0,0 +1,89 @@ +abstract type RouteCardinality end + +""" + ManyToOneVector() + +Route cardinality that materializes one value per resolved producer. +""" +struct ManyToOneVector <: RouteCardinality end + +""" + ManyToOneAggregate(reducer=sum) + +Route cardinality that reduces resolved producer values to one scalar. +""" +struct ManyToOneAggregate{F} <: RouteCardinality + reducer::F +end + +ManyToOneAggregate() = ManyToOneAggregate(sum) + +""" + OneToManyBroadcast() + SpatialSample() + SpatialScatterAdd() + +Reserved route cardinalities for the MTG/spatial domain runner. +""" +struct OneToManyBroadcast <: RouteCardinality end +struct SpatialSample <: RouteCardinality end +struct SpatialScatterAdd <: RouteCardinality end + +""" + DomainRouteTarget(domain; scale=:Default, var, process=nothing) + +Target of an explicit cross-domain route. +""" +struct DomainRouteTarget + domain::Symbol + scale::Symbol + var::Symbol + process::Union{Nothing,Symbol} +end + +function Base.show(io::IO, target::DomainRouteTarget) + terms = ["domain=:$(target.domain)"] + target.scale == :Default || push!(terms, "scale=:$(target.scale)") + push!(terms, "var=:$(target.var)") + isnothing(target.process) || push!(terms, "process=:$(target.process)") + print(io, "DomainRouteTarget(", join(terms, ", "), ")") +end + +function DomainRouteTarget( + domain::Union{Symbol,AbstractString}; + scale::Union{Symbol,AbstractString}=:Default, + var, + process=nothing, +) + return DomainRouteTarget( + Symbol(domain), + Symbol(scale), + Symbol(var), + isnothing(process) ? nothing : Symbol(process), + ) +end + +""" + Route(from, to; cardinality=ManyToOneVector(), policy=nothing) + Route(; from, to, cardinality=ManyToOneVector(), policy=nothing) + +Explicit cross-domain route materialized into a target domain status before +that domain runs. +""" +struct Route{F,T,C<:RouteCardinality,P<:SchedulePolicy} + from::F + to::T + cardinality::C + policy::P +end + +function Route(from::AllDomains, to::DomainRouteTarget; cardinality::RouteCardinality=ManyToOneVector(), policy=nothing) + route_policy = isnothing(policy) ? from.policy : _as_schedule_policy(policy; context="Route policy") + isnothing(from.var) && error( + "Route source `AllDomains(...)` must declare `var=:source_variable` so the runtime knows what to materialize." + ) + return Route(from, to, cardinality, route_policy) +end + +Route(; from, to, cardinality::RouteCardinality=ManyToOneVector(), policy=nothing) = + Route(from, to; cardinality=cardinality, policy=policy) diff --git a/src/evaluation/fit.jl b/src/evaluation/fit.jl index 7998d6a5d..873127244 100644 --- a/src/evaluation/fit.jl +++ b/src/evaluation/fit.jl @@ -29,7 +29,7 @@ and `Ri_PAR_f`. # Including example processes and models: using PlantSimEngine.Examples; -m = ModelList(Beer(0.6), status=(LAI=2.0,)) +m = ModelMapping(Beer(0.6), status=(LAI=2.0,)) meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0) run!(m, meteo) df = DataFrame(aPPFD=m[:aPPFD][1], LAI=m.status.LAI[1], Ri_PAR_f=meteo.Ri_PAR_f[1]) @@ -40,4 +40,4 @@ Note that this is a dummy example to show that the fitting method works, as we s using the Beer-Lambert law with a value of `k=0.6`, and then use the simulated aPPFD to fit the `k` parameter again, which gives the same value as the one used on the simulation. """ -function fit end \ No newline at end of file +function fit end diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 8b826680d..5ba81b4f1 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -23,7 +23,7 @@ struct GraphSimulation{T,S,U,O,V,TS,MS} graph::T statuses::S status_templates::Dict{Symbol,Dict{Symbol,Any}} - reverse_multiscale_mapping::Dict{Symbol,Dict{Symbol,Dict{Symbol,Any}}} + reverse_multiscale_mapping::ReverseMultiscaleMapping var_need_init::Dict{Symbol,V} dependency_graph::DependencyGraph models::Dict{Symbol,U} @@ -141,17 +141,10 @@ function convert_outputs(outs::Dict{Symbol,O} where O, sink; refvectors=false, n return ret end -# TODO adapt these to new output structure or remove them -function outputs(outs::Dict{Symbol,O} where O, key::Symbol) - Tables.columns(convert_outputs(outs, Vector{NamedTuple}))[key] -end - -function outputs(outs::Dict{Symbol,O} where O, i::T) where {T<:Integer} - Tables.columns(convert_outputs(outs, Vector{NamedTuple}))[i] -end - -# ModelLists now return outputs as a TimeStepTable{Status}, conversion is straightforward +# Single-scale mappings return outputs as a TimeStepTable{Status}, conversion is straightforward function convert_outputs(out::TimeStepTable{T} where T, sink) - @assert Tables.istable(sink) "The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)" + if !Tables.istable(sink) + error("The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)") + end return sink(out) end diff --git a/src/mtg/ModelSpec.jl b/src/mtg/ModelSpec.jl index f461515ec..a4a4eeae0 100644 --- a/src/mtg/ModelSpec.jl +++ b/src/mtg/ModelSpec.jl @@ -1,5 +1,5 @@ """ - ModelSpec(model; multiscale=nothing, timestep=nothing, input_bindings=NamedTuple(), meteo_bindings=NamedTuple(), meteo_window=nothing, output_routing=NamedTuple(), scope=:global) + ModelSpec(model; name=nothing, applies_to=nothing, inputs=NamedTuple(), calls=NamedTuple(), environment=nothing, multiscale=nothing, timestep=nothing, input_bindings=NamedTuple(), meteo_bindings=NamedTuple(), meteo_window=nothing, output_routing=NamedTuple(), scope=:global, updates=()) User-side model configuration wrapper for mapping/model list composition. @@ -7,8 +7,13 @@ User-side model configuration wrapper for mapping/model list composition. This allows modelers to publish reusable models while users decide how models are coupled in their simulation setup. """ -struct ModelSpec{M,MS,TS,IB,MB,MW,OR,SC} +struct ModelSpec{M,N,AT,IN,CA,EV,MS,TS,IB,MB,MW,OR,SC,UP} model::M + name::N + applies_to::AT + inputs::IN + calls::CA + environment::EV multiscale::MS timestep::TS input_bindings::IB @@ -16,6 +21,33 @@ struct ModelSpec{M,MS,TS,IB,MB,MW,OR,SC} meteo_window::MW output_routing::OR scope::SC + updates::UP +end + +""" + Updates(vars...; after=nothing) + +Scenario-level declaration that a model updates variables which may also be +computed by another model at the same scale. + +`after` is intentionally mapping-level metadata: the model implementation stays +reusable, while the simulation setup can declare ordering constraints that only +exist in this coupling. +""" +struct Updates{V,A} + variables::V + after::A +end + +function Updates(vars::Vararg{Union{Symbol,AbstractString}}; after=nothing) + isempty(vars) && error("Updates(...) requires at least one variable.") + return Updates(Tuple(Symbol(v) for v in vars), _normalize_update_after(after)) +end + +function _normalize_update_after(after) + isnothing(after) && return () + after isa Union{Symbol,AbstractString} && return (Symbol(after),) + return Tuple(Symbol(v) for v in after) end function _normalize_multiscale_mapping(model::AbstractModel, mapped_variables) @@ -24,60 +56,190 @@ function _normalize_multiscale_mapping(model::AbstractModel, mapped_variables) return mapped_variables_(mapped) end +_normalize_updates(updates::Updates) = (updates,) + +function _normalize_updates(updates::Tuple) + all(update -> update isa Updates, updates) || error( + "Unsupported updates tuple. Use `Updates(:var; after=:process)` entries." + ) + return updates +end + +function _normalize_updates(updates::AbstractVector) + all(update -> update isa Updates, updates) || error( + "Unsupported updates vector. Use `Updates(:var; after=:process)` entries." + ) + return Tuple(updates) +end + +function _normalize_updates(updates) + updates == NamedTuple() && return () + updates == () && return () + error( + "Unsupported updates metadata `$(updates)` of type `$(typeof(updates))`. ", + "Use `Updates(:var; after=:process)` or a tuple/vector of `Updates`." + ) +end + function ModelSpec( model::AbstractModel; + name=nothing, + applies_to=nothing, + inputs=NamedTuple(), + calls=NamedTuple(), + environment=nothing, multiscale=nothing, timestep=nothing, input_bindings=NamedTuple(), meteo_bindings=NamedTuple(), meteo_window=nothing, output_routing=NamedTuple(), - scope=:global + scope=:global, + updates=() ) base_model = model - base_multiscale = multiscale - if model isa MultiScaleModel - base_model = model_(model) - base_multiscale === nothing && (base_multiscale = mapped_variables_(model)) - end - - normalized_multiscale = _normalize_multiscale_mapping(base_model, base_multiscale) + normalized_name = _normalize_application_name(name) + default_inputs = _model_default_value_inputs(base_model) + normalized_inputs = _merge_value_inputs(default_inputs, _normalize_application_bindings(inputs)) + default_calls = _model_default_model_calls(base_model) + normalized_calls = _merge_value_inputs(default_calls, _normalize_application_bindings(calls)) + derived_multiscale = _legacy_multiscale_from_value_inputs(normalized_inputs, base_model) + combined_multiscale = _merge_legacy_multiscale(multiscale, derived_multiscale) + normalized_multiscale = _normalize_multiscale_mapping(base_model, combined_multiscale) normalized_input_bindings = _normalize_input_bindings(input_bindings) normalized_meteo_bindings = _normalize_meteo_bindings(meteo_bindings) normalized_meteo_window = _normalize_meteo_window(meteo_window) normalized_output_routing = _normalize_output_routing(output_routing) normalized_scope = _normalize_scope_selector(scope) - return ModelSpec{typeof(base_model),typeof(normalized_multiscale),typeof(timestep),typeof(normalized_input_bindings),typeof(normalized_meteo_bindings),typeof(normalized_meteo_window),typeof(normalized_output_routing),typeof(normalized_scope)}( + normalized_updates = _normalize_updates(updates) + return ModelSpec{typeof(base_model),typeof(normalized_name),typeof(applies_to),typeof(normalized_inputs),typeof(normalized_calls),typeof(environment),typeof(normalized_multiscale),typeof(timestep),typeof(normalized_input_bindings),typeof(normalized_meteo_bindings),typeof(normalized_meteo_window),typeof(normalized_output_routing),typeof(normalized_scope),typeof(normalized_updates)}( base_model, + normalized_name, + applies_to, + normalized_inputs, + normalized_calls, + environment, normalized_multiscale, timestep, normalized_input_bindings, normalized_meteo_bindings, normalized_meteo_window, normalized_output_routing, - normalized_scope + normalized_scope, + normalized_updates + ) +end + +function ModelSpec( + model::MultiScaleModel; + name=nothing, + applies_to=nothing, + inputs=NamedTuple(), + calls=NamedTuple(), + environment=nothing, + multiscale=nothing, + timestep=nothing, + input_bindings=NamedTuple(), + meteo_bindings=NamedTuple(), + meteo_window=nothing, + output_routing=NamedTuple(), + scope=:global, + updates=() +) + base_multiscale = isnothing(multiscale) ? mapped_variables_(model) : multiscale + return ModelSpec( + model_(model); + name=name, + applies_to=applies_to, + inputs=inputs, + calls=calls, + environment=environment, + multiscale=base_multiscale, + timestep=timestep, + input_bindings=input_bindings, + meteo_bindings=meteo_bindings, + meteo_window=meteo_window, + output_routing=output_routing, + scope=scope, + updates=updates ) end function ModelSpec( spec::ModelSpec; model=spec.model, + name=spec.name, + applies_to=spec.applies_to, + inputs=spec.inputs, + calls=spec.calls, + environment=spec.environment, multiscale=spec.multiscale, timestep=spec.timestep, input_bindings=spec.input_bindings, meteo_bindings=spec.meteo_bindings, meteo_window=spec.meteo_window, output_routing=spec.output_routing, - scope=spec.scope + scope=spec.scope, + updates=spec.updates ) - ModelSpec(model; multiscale=multiscale, timestep=timestep, input_bindings=input_bindings, meteo_bindings=meteo_bindings, meteo_window=meteo_window, output_routing=output_routing, scope=scope) + ModelSpec(model; name=name, applies_to=applies_to, inputs=inputs, calls=calls, environment=environment, multiscale=multiscale, timestep=timestep, input_bindings=input_bindings, meteo_bindings=meteo_bindings, meteo_window=meteo_window, output_routing=output_routing, scope=scope, updates=updates) end as_model_spec(spec::ModelSpec) = spec as_model_spec(model::AbstractModel) = ModelSpec(model) as_model_spec(model::MultiScaleModel) = ModelSpec(model_(model); multiscale=mapped_variables_(model)) +""" + with_name(model_or_spec, name) + +Return a `ModelSpec` with an explicit model-application name. +""" +function with_name(model_or_spec, name) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; name=_normalize_application_name(name)) +end + +""" + with_applies_to(model_or_spec, selector) + +Return a `ModelSpec` with an explicit model-application target selector. +""" +function with_applies_to(model_or_spec, selector) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; applies_to=selector) +end + +""" + with_inputs(model_or_spec, bindings) + +Return a `ModelSpec` with unified scene/object value-input bindings. +""" +function with_inputs(model_or_spec, bindings) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; inputs=_normalize_application_bindings(bindings)) +end + +""" + with_calls(model_or_spec, bindings) + +Return a `ModelSpec` with unified scene/object manual model-call bindings. +""" +function with_calls(model_or_spec, bindings) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; calls=_normalize_application_bindings(bindings)) +end + +""" + with_environment(model_or_spec, environment) + +Return a `ModelSpec` with scene/object environment configuration metadata. +""" +function with_environment(model_or_spec, environment) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; environment=environment) +end + """ with_multiscale(model_or_spec, mapped_variables) @@ -148,6 +310,18 @@ function with_scope(model_or_spec, scope) return ModelSpec(spec; scope=_normalize_scope_selector(scope)) end +""" + with_updates(model_or_spec, updates) + +Return a `ModelSpec` with explicit variable-update metadata. +""" +function with_updates(model_or_spec, updates) + spec = as_model_spec(model_or_spec) + return ModelSpec(spec; updates=(spec.updates..., _normalize_updates(updates)...)) +end + +(updates::Updates)(model_or_spec) = with_updates(model_or_spec, updates) + function _normalize_input_binding(binding) if binding isa NamedTuple return haskey(binding, :policy) ? binding : (; binding..., policy=HoldLast()) @@ -167,7 +341,9 @@ function _normalize_input_bindings(bindings::NamedTuple) return (; normalized...) end -_normalize_input_bindings(bindings) = bindings +function _normalize_input_bindings(bindings) + error("Unsupported InputBindings value `$(bindings)` of type `$(typeof(bindings))`. Use a NamedTuple, e.g. `InputBindings(; x=(process=:producer, var=:y))`.") +end function _normalize_meteo_binding(binding) if binding isa DataType @@ -197,7 +373,9 @@ function _normalize_meteo_bindings(bindings::NamedTuple) return (; normalized...) end -_normalize_meteo_bindings(bindings) = bindings +function _normalize_meteo_bindings(bindings) + error("Unsupported MeteoBindings value `$(bindings)` of type `$(typeof(bindings))`. Use a NamedTuple, e.g. `MeteoBindings(; T=MeanReducer())`.") +end function _normalize_meteo_window(window) if isnothing(window) @@ -231,15 +409,65 @@ function _normalize_output_routing(routing::NamedTuple) return (; normalized...) end -_normalize_output_routing(routing) = routing +function _normalize_output_routing(routing) + error("Unsupported OutputRouting value `$(routing)` of type `$(typeof(routing))`. Use a NamedTuple, e.g. `OutputRouting(; x=:stream_only)`.") +end function _normalize_scope_selector(scope) if scope isa AbstractString - return Symbol(scope) + error("String scope selectors are not supported. Use symbols such as `:global`, `:plant`, `:scene`, or `:self`.") end return scope end +""" + AppliesTo(selector) + +Pipe-style transform that sets the object selector where a model application +runs in the unified scene/object API. +""" +AppliesTo(selector) = x -> with_applies_to(x, selector) + +""" + Inputs(bindings...) + Inputs(; kwargs...) + +Pipe-style transform that sets value-input bindings in the unified +scene/object API. +""" +Inputs(bindings::Pair...) = x -> with_inputs(x, bindings) +Inputs(bindings::NamedTuple) = x -> with_inputs(x, bindings) +Inputs(; kwargs...) = Inputs((; kwargs...)) + +""" + Calls(bindings...) + Calls(; kwargs...) + +Pipe-style transform that sets manual model-call bindings in the unified +scene/object API. +""" +Calls(bindings::Pair...) = x -> with_calls(x, bindings) +Calls(bindings::NamedTuple) = x -> with_calls(x, bindings) +Calls(; kwargs...) = Calls((; kwargs...)) + +""" + TimeStep(timestep) + +Pipe-style transform that sets a user-selected timestep in the unified +scene/object API. This is the breaking-design name for `TimeStepModel(...)`. +""" +TimeStep(timestep) = x -> with_timestep(x, timestep) + +""" + Environment(config) + Environment(; kwargs...) + +Pipe-style transform that stores scene/object environment configuration +metadata on a `ModelSpec`. +""" +Environment(config) = x -> with_environment(x, config isa EnvironmentConfig ? config : EnvironmentConfig(config)) +Environment(; kwargs...) = Environment((; kwargs...)) + """ MultiScaleModel(mapped_variables) @@ -389,9 +617,9 @@ multi-rate simulations. # Arguments - `scope`: one of: - - selector symbols/strings: `:global`, `:plant`, `:scene`, `:self`, + - selector symbols: `:global`, `:plant`, `:scene`, `:self`, - a concrete `ScopeId`, - - a callable returning a scope selector/id at runtime. + - a callable returning a scope selector symbol or `ScopeId` at runtime. # Example ```julia @@ -408,3 +636,47 @@ get_status(m::ModelSpec) = nothing get_mapped_variables(m::ModelSpec) = mapped_variables_(m) process(m::ModelSpec) = process(model_(m)) timestep(m::ModelSpec) = m.timestep +inputs_(m::ModelSpec) = inputs_(model_(m)) +outputs_(m::ModelSpec) = outputs_(model_(m)) + +function _model_spec_dependency_selector(dep_name::Symbol, selector) + return selector +end + +function _model_spec_dependency_selector(dep_name::Symbol, selector::Input) + return nothing +end + +function _normalize_model_spec_dependencies(deps::NamedTuple) + normalized = Pair{Symbol,Any}[] + for (dep_name, selector) in pairs(deps) + selector isa Union{Input,Call} && continue + normalized_selector = _model_spec_dependency_selector(dep_name, selector) + isnothing(normalized_selector) && continue + push!(normalized, dep_name => normalized_selector) + end + return (; normalized...) +end + +function _model_spec_call_dependencies(spec::ModelSpec) + calls = model_calls(spec) + calls isa NamedTuple || return NamedTuple() + normalized = Pair{Symbol,Any}[] + for (dep_name, selector) in pairs(calls) + push!(normalized, dep_name => _model_spec_dependency_selector(dep_name, selector)) + end + return (; normalized...) +end + +function dep(m::ModelSpec) + model_deps = _normalize_model_spec_dependencies(dep(model_(m))) + call_deps = _model_spec_call_dependencies(m) + return (; pairs(model_deps)..., pairs(call_deps)...) +end +init_variables(m::ModelSpec; verbose::Bool=true) = init_variables(model_(m); verbose=verbose) +meteo_inputs_(m::ModelSpec) = meteo_inputs_(model_(m)) +meteo_outputs_(m::ModelSpec) = meteo_outputs_(model_(m)) + +function run!(m::ModelSpec, models, status, meteo, constants=nothing, extra=nothing) + return run!(model_(m), models, status, meteo, constants, extra) +end diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index a5535af6f..7b6d1b66b 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -7,7 +7,7 @@ model and the nodes symbols from which the values are taken from. # Arguments - `model<:AbstractModel`: the model to make multi-scale -- `mapped_variables<:Vector{Pair{Symbol,Union{Symbol,AbstractString,Vector{<:Union{Symbol,AbstractString}}}}}`: +- `mapped_variables<:Vector{Pair{Symbol,Union{Symbol,Vector{Symbol}}}}`: a vector of pairs of model variables and source scale declarations. The mapped_variables argument can be of the form: @@ -18,7 +18,7 @@ The mapped_variables argument can be of the form: 4. `[:variable_name => (:Plant => :variable_name_in_plant_scale)]`: We take one value from another variable name in the Plant node 5. `[:variable_name => [:Leaf => :variable_name_1, :Internode => :variable_name_2]]`: We take a vector of values from the Leaf and Internode nodes with different names 6. `[PreviousTimeStep(:variable_name) => ...]`: We flag the variable to be initialized with the value from the previous time step, and we do not use it to build the dep graph -7. `[:variable_name => (Symbol() => :variable_name_from_another_model)]`: We take the value from another model at the same scale but rename it +7. `[:variable_name => (SameScale() => :variable_name_from_another_model)]`: We take the value from another model at the same scale but rename it 8. `[PreviousTimeStep(:variable_name),]`: We just flag the variable as a PreviousTimeStep to not use it to build the dep graph Details about the different forms: @@ -101,15 +101,7 @@ julia> PlantSimEngine.model_(multiscale_model) ToyCAllocationModel() ``` """ -struct MultiScaleModel{ - T<:AbstractModel, - V<:AbstractVector{ - Pair{ - A, - Union{Pair{Symbol,Symbol},Vector{Pair{Symbol,Symbol}}} - } - } where {A<:Union{Symbol,PreviousTimeStep}} -} +struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector} model::T mapped_variables::V @@ -135,11 +127,11 @@ struct MultiScaleModel{ # 4. `[:variable_name => (:Plant => :variable_name_in_plant_scale)]` # We take one value from another variable name in the Plant node # 5. `[:variable_name => [:Leaf => :variable_name_1, :Internode => :variable_name_2]]` # We take a vector of values from the Leaf and Internode nodes with different names # 6. `[PreviousTimeStep(:variable_name) => ...]` # We flag the variable to be initialized with the value from the previous time step, and we do not use it to build the dep graph - # 7. `[:variable_name => (Symbol("") => :variable_name_from_another_model)]` # We take the value from another model at the same scale but rename it + # 7. `[:variable_name => (SameScale() => :variable_name_from_another_model)]` # We take the value from another model at the same scale but rename it # 8. `[PreviousTimeStep(:variable_name),]` # We just flag the variable as a PreviousTimeStep to not use it to build the dep graph process_ = process(model) - unfolded_mapping = Pair{Union{Symbol,PreviousTimeStep},Union{Pair{Symbol,Symbol},Vector{Pair{Symbol,Symbol}}}}[] + unfolded_mapping = Pair{Union{Symbol,PreviousTimeStep},Any}[] for i in mapped_variables push!(unfolded_mapping, _get_var(isa(i, PreviousTimeStep) ? i : Pair(i.first, i.second), process_)) # Note: We are using Pair(i.first, i.second) to make sure the Pair is specialized enough, because sometimes the vector in the mapping made the Pair not specialized enough e.g. [:v1 => :S => :v2,:v3 => :S] makes the pairs `Pair{Symbol, Any}`. @@ -154,35 +146,25 @@ function _get_var(i::Pair{Symbol,Any}, proc::Symbol=:unknown) return _get_var(first(i) => last(i), proc) end -@noinline function _warn_multiscale_model_string_scale() - Base.depwarn( - "String scale names in `MultiScaleModel(mapped_variables=...)` are deprecated and will be removed in a future release. Use Symbol scales, e.g. `:Leaf` instead of `\"Leaf\"`.", - :MultiScaleModel - ) +function _normalize_mapped_scale(scale::Symbol) + scale === Symbol("") && error("`Symbol(\"\")` same-scale mappings are removed. Use `SameScale()` instead.") + return scale end - -_normalize_mapped_scale(scale::Symbol) = scale -function _normalize_mapped_scale(scale::AbstractString) - _warn_multiscale_model_string_scale() - return Symbol(scale) -end - -# Case 1: [:variable_name => :Plant] (deprecated, coerced to symbol scale) -function _get_var(i::Pair{Symbol,S}, proc::Symbol=:unknown) where {S<:AbstractString} - scale = _normalize_mapped_scale(last(i)) - return first(i) => scale => first(i) +_normalize_mapped_scale(scale::SameScale) = scale +function _normalize_mapped_scale(scale) + error("Mapped scale names must be `Symbol`s, got `$(typeof(scale))` for `$(repr(scale))`.") end -function _get_var(i::Pair{Symbol,Pair{S,Symbol}}, proc::Symbol=:unknown) where {S<:Union{AbstractString,Symbol}} +function _get_var(i::Pair{Symbol,Pair{T,Symbol}}, proc::Symbol=:unknown) where {T<:Union{Symbol,SameScale}} return first(i) => (_normalize_mapped_scale(first(last(i))) => last(last(i))) end -function _get_var(i::Pair{Symbol,T}, proc::Symbol=:unknown) where {T<:AbstractVector{<:Union{AbstractString,Symbol}}} +function _get_var(i::Pair{Symbol,T}, proc::Symbol=:unknown) where {T<:AbstractVector{<:Symbol}} return first(i) => [_normalize_mapped_scale(scale) => first(i) for scale in last(i)] end # Case 5: [:variable_name => [:Leaf => :variable_name_1, :Internode => :variable_name_2]] -function _get_var(i::Pair{Symbol,T}, proc::Symbol=:unknown) where {T<:AbstractVector{<:Pair{<:Union{AbstractString,Symbol},Symbol}}} +function _get_var(i::Pair{Symbol,T}, proc::Symbol=:unknown) where {T<:AbstractVector{<:Pair{Symbol,Symbol}}} return first(i) => [_normalize_mapped_scale(first(mapping)) => last(mapping) for mapping in last(i)] end @@ -199,7 +181,7 @@ end # Case 8: [PreviousTimeStep(:variable_name),] function _get_var(i::PreviousTimeStep, proc::Symbol=:unknown) - return PreviousTimeStep(i.variable, proc) => Symbol("") => i.variable + return PreviousTimeStep(i.variable, proc) => SameScale() => i.variable end diff --git a/src/mtg/add_organ.jl b/src/mtg/add_organ.jl index 784127f89..6d448b2b2 100644 --- a/src/mtg/add_organ.jl +++ b/src/mtg/add_organ.jl @@ -4,7 +4,7 @@ Add an organ to the graph, automatically taking care of initialising the status of the organ (multiscale-)variables. This function should be called from a model that implements organ emergence, for example in function of thermal time. - + # Arguments * `node`: the node to which the organ is added (the parent organ of the new organ) @@ -20,7 +20,7 @@ This function should be called from a model that implements organ emergence, for * `attributes`: the attributes of the new node. If not provided, an empty dictionary is used. * `check`: a boolean indicating if variables initialisation should be checked. Passed to `init_node_status!`. -# Returns +# Returns * `status`: the status of the new node @@ -32,6 +32,199 @@ or the `test-mtg-dynamic.jl` test file for an example usage. function add_organ!(node::MultiScaleTreeGraph.Node, sim_object, link, symbol, scale; index=0, id=MultiScaleTreeGraph.new_id(MultiScaleTreeGraph.get_root(node)), attributes=Dict{Symbol,Any}(), check=true) new_node = MultiScaleTreeGraph.Node(id, node, MultiScaleTreeGraph.NodeMTG(link, symbol, index, scale), attributes) st = init_node_status!(new_node, sim_object.statuses, sim_object.status_templates, sim_object.reverse_multiscale_mapping, sim_object.var_need_init, check=check) + reindex_runtime_topology!(sim_object) return st -end \ No newline at end of file +end + +function _delete_ref_from_refvector!(rv::RefVector, ref) + filter!(stored_ref -> stored_ref !== ref, parent(rv)) + return rv +end + +function _remove_status_from_scale!(statuses::Dict, scale::Symbol, st::Status, nid::Int) + haskey(statuses, scale) || return nothing + deleteat!(statuses[scale], findall(candidate -> candidate === st || node_id(candidate.node) == nid, statuses[scale])) + return nothing +end + +function _remove_reverse_refs_for_status!(sim_object, node_scale::Symbol, st::Status) + haskey(sim_object.reverse_multiscale_mapping, node_scale) || return nothing + for (target_scale, vars) in sim_object.reverse_multiscale_mapping[node_scale] + haskey(sim_object.status_templates, target_scale) || continue + target_template = sim_object.status_templates[target_scale] + for (source_var, target_var_) in vars + source_var in propertynames(st) || continue + target_var = target_var_ isa PreviousTimeStep ? target_var_.variable : target_var_ + haskey(target_template, target_var) || continue + target_value = target_template[target_var] + target_value isa RefVector || continue + _delete_ref_from_refvector!(target_value, refvalue(st, source_var)) + end + end + return nothing +end + +function _remove_temporal_state_for_node!(sim_object, nid::Int) + temporal = temporal_state(sim_object) + for key in collect(keys(temporal.caches)) + key.node_id == nid && delete!(temporal.caches, key) + end + for key in collect(keys(temporal.streams)) + key.node_id == nid && delete!(temporal.streams, key) + end + return nothing +end + +function _subtree_node_ids(node::MultiScaleTreeGraph.Node) + ids = Int[] + MultiScaleTreeGraph.traverse!(node) do subtree_node + push!(ids, node_id(subtree_node)) + end + return ids +end + +function _remove_temporal_state_for_nodes!(sim_object, node_ids) + for nid in node_ids + _remove_temporal_state_for_node!(sim_object, nid) + end + return nothing +end + +function _status_node_registered(sim_object, node::MultiScaleTreeGraph.Node) + node_scale = symbol(node) + haskey(sim_object.statuses, node_scale) || return false + nid = node_id(node) + return any(st -> hasproperty(st, :node) && node_id(st.node) == nid && st.node === node, sim_object.statuses[node_scale]) +end + +function _is_descendant_node(candidate::MultiScaleTreeGraph.Node, ancestor::MultiScaleTreeGraph.Node) + current = parent(candidate) + while !isnothing(current) + current === ancestor && return true + current = parent(current) + end + return false +end + +function _children_without_node(parent_node::MultiScaleTreeGraph.Node, node::MultiScaleTreeGraph.Node) + children_without_node = empty(AbstractTrees.children(parent_node)) + for child in AbstractTrees.children(parent_node) + child === node || push!(children_without_node, child) + end + return children_without_node +end + +function _repair_reparent_child_links!( + node::MultiScaleTreeGraph.Node, + old_parent, + new_parent::MultiScaleTreeGraph.Node, +) + if !isnothing(old_parent) && old_parent !== new_parent + MultiScaleTreeGraph.rechildren!(old_parent, _children_without_node(old_parent, node)) + end + + new_children = _children_without_node(new_parent, node) + push!(new_children, node) + MultiScaleTreeGraph.rechildren!(new_parent, new_children) + return nothing +end + +""" + remove_organ!(node::MultiScaleTreeGraph.Node, sim_object; attribute_name=:plantsimengine_status, recursive=false) + +Remove a simulated organ from an active [`GraphSimulation`](@ref). + +The wrapper updates PlantSimEngine runtime state before delegating to +`MultiScaleTreeGraph.delete_node!`: it removes the node status from +`sim_object.statuses`, removes references from downstream `RefVector`s, clears +temporal caches/streams for the removed node, and then deletes the MTG node. + +Only leaf/terminal nodes are removed by default. Pass `recursive=true` to delete +an internal node and its whole subtree. Reparenting children is intentionally not +handled here because it requires caller-specific biological and topological +policy. +""" +function remove_organ!(node::MultiScaleTreeGraph.Node, sim_object; attribute_name=:plantsimengine_status, recursive=false) + children = collect(AbstractTrees.children(node)) + if !recursive + isempty(children) || error( + "remove_organ!(...; recursive=false) only supports leaf/terminal MTG nodes. ", + "Pass `recursive=true` to delete node $(node_id(node)) and its descendants, ", + "or move descendants first." + ) + else + for child in children + remove_organ!(child, sim_object; attribute_name=attribute_name, recursive=true) + end + end + + haskey(node, attribute_name) || error( + "Cannot remove MTG node $(node_id(node)) ($(symbol(node))) from PlantSimEngine runtime: ", + "the node has no `$(attribute_name)` status." + ) + + st = node[attribute_name] + st isa Status || error( + "Cannot remove MTG node $(node_id(node)) ($(symbol(node))) from PlantSimEngine runtime: ", + "`$(attribute_name)` is not a Status." + ) + + nid = node_id(node) + node_scale = symbol(node) + _remove_reverse_refs_for_status!(sim_object, node_scale, st) + _remove_status_from_scale!(sim_object.statuses, node_scale, st, nid) + _remove_temporal_state_for_node!(sim_object, nid) + pop!(node, attribute_name) + deleted = MultiScaleTreeGraph.delete_node!(node) + reindex_runtime_topology!(sim_object) + return deleted +end + +""" + reparent_organ!(node::MultiScaleTreeGraph.Node, new_parent::MultiScaleTreeGraph.Node, sim_object; attribute_name=:plantsimengine_status) + +Move an already-simulated MTG node under another already-simulated parent in the +same active [`GraphSimulation`](@ref). + +The node status and downstream `RefVector`s keep pointing to the same node +object, so no status rewiring is needed when the subtree remains in the same +simulation. This wrapper validates that both nodes are registered in +PlantSimEngine runtime state and rejects moves that would create a cycle. +""" +function reparent_organ!( + node::MultiScaleTreeGraph.Node, + new_parent::MultiScaleTreeGraph.Node, + sim_object; + attribute_name=:plantsimengine_status, +) + node === new_parent && error("Cannot reparent MTG node $(node_id(node)) to itself.") + _is_descendant_node(new_parent, node) && error( + "Cannot reparent MTG node $(node_id(node)) under descendant node $(node_id(new_parent)); ", + "this would create a cycle." + ) + haskey(node, attribute_name) || error( + "Cannot reparent MTG node $(node_id(node)) ($(symbol(node))) in PlantSimEngine runtime: ", + "the node has no `$(attribute_name)` status." + ) + haskey(new_parent, attribute_name) || error( + "Cannot reparent MTG node $(node_id(node)) under node $(node_id(new_parent)) ($(symbol(new_parent))) ", + "in PlantSimEngine runtime: the new parent has no `$(attribute_name)` status." + ) + _status_node_registered(sim_object, node) || error( + "Cannot reparent MTG node $(node_id(node)) ($(symbol(node))): ", + "it is not registered in this GraphSimulation." + ) + _status_node_registered(sim_object, new_parent) || error( + "Cannot reparent MTG node $(node_id(node)) under node $(node_id(new_parent)) ($(symbol(new_parent))): ", + "the new parent is not registered in this GraphSimulation." + ) + + old_parent = parent(node) + moved_node_ids = _subtree_node_ids(node) + MultiScaleTreeGraph.reparent!(node, new_parent) + _repair_reparent_child_links!(node, old_parent, new_parent) + _remove_temporal_state_for_nodes!(sim_object, moved_node_ids) + reindex_runtime_topology!(sim_object) + return node +end diff --git a/src/mtg/initialisation.jl b/src/mtg/initialisation.jl index 98aa612c6..f15a6a523 100644 --- a/src/mtg/initialisation.jl +++ b/src/mtg/initialisation.jl @@ -48,6 +48,7 @@ function init_statuses(mtg, mapping, dependency_graph; type_promotion=nothing, v MultiScaleTreeGraph.traverse!(mtg) do node # e.g.: node = MultiScaleTreeGraph.get_node(mtg, 5) init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init, type_promotion, check=check) end + reindex_runtime_topology!(statuses, mapped_vars, reverse_multiscale_mapping) return (; statuses, mapped_vars, reverse_multiscale_mapping, vars_need_init) end @@ -136,10 +137,12 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi error("Failed to convert variable `$(var)` in MTG node $(node_id(node)) ($(symbol(node))) from type `$(typeof(node[var]))` to type `$(eltype(st_template[var]))`: $(e)") end end - @assert typeof(node_var) == eltype(st_template[var]) string( - "Initializing variable `$(var)` using MTG node $(node_id(node)) ($(symbol(node))): expected type $(eltype(st_template[var])), found $(typeof(node_var)). ", - "Please check the type of the variable in the MTG, and make it a $(eltype(st_template[var])) by updating the model, or by using `type_promotion`." - ) + if typeof(node_var) != eltype(st_template[var]) + error( + "Initializing variable `$(var)` using MTG node $(node_id(node)) ($(symbol(node))): expected type $(eltype(st_template[var])), found $(typeof(node_var)). ", + "Please check the type of the variable in the MTG, and make it a $(eltype(st_template[var])) by updating the model, or by using `type_promotion`." + ) + end st_template[var] = node_var # NB: the variable is not a reference to the value in the MTG, but a copy of it. # This is because we can't reference a value in a Dict. If we need a ref, the user can use a RefValue in the MTG directly, @@ -170,6 +173,83 @@ function init_node_status!(node, statuses, mapped_vars, reverse_multiscale_mappi return st end +function _status_sort_key(st::Status) + hasproperty(st, :node) || return typemax(Int) + return node_id(st.node) +end + +function _sort_statuses_by_node_id!(statuses) + for statuses_at_scale in values(statuses) + sort!(statuses_at_scale; by=_status_sort_key) + end + return statuses +end + +function _empty_reverse_refvectors!(status_templates, reverse_multiscale_mapping) + for (_, target_scales) in reverse_multiscale_mapping + for (target_scale, vars) in target_scales + haskey(status_templates, target_scale) || continue + target_template = status_templates[target_scale] + for (_, target_var_) in vars + target_var = target_var_ isa PreviousTimeStep ? target_var_.variable : target_var_ + haskey(target_template, target_var) || continue + target_value = target_template[target_var] + target_value isa RefVector || continue + empty!(target_value) + end + end + end + return nothing +end + +function _rebuild_reverse_refvectors!(statuses, status_templates, reverse_multiscale_mapping) + _empty_reverse_refvectors!(status_templates, reverse_multiscale_mapping) + refs_by_target = Dict{Tuple{Symbol,Symbol},Vector{Tuple{Int,Base.RefValue}}}() + for (source_scale, statuses_at_scale) in statuses + haskey(reverse_multiscale_mapping, source_scale) || continue + for st in statuses_at_scale + for (target_scale, vars) in reverse_multiscale_mapping[source_scale] + haskey(status_templates, target_scale) || continue + target_template = status_templates[target_scale] + for (source_var, target_var_) in vars + source_var in propertynames(st) || continue + target_var = target_var_ isa PreviousTimeStep ? target_var_.variable : target_var_ + haskey(target_template, target_var) || continue + target_value = target_template[target_var] + target_value isa RefVector || continue + push!( + get!(refs_by_target, (target_scale, target_var), Tuple{Int,Base.RefValue}[]), + (_status_sort_key(st), refvalue(st, source_var)), + ) + end + end + end + end + for ((target_scale, target_var), refs) in refs_by_target + target_value = status_templates[target_scale][target_var] + sort!(refs; by=first) + for (_, ref) in refs + push!(target_value, ref) + end + end + return nothing +end + +function reindex_runtime_topology!(statuses, status_templates, reverse_multiscale_mapping) + _sort_statuses_by_node_id!(statuses) + _rebuild_reverse_refvectors!(statuses, status_templates, reverse_multiscale_mapping) + return nothing +end + +function reindex_runtime_topology!(sim_object) + reindex_runtime_topology!( + sim_object.statuses, + sim_object.status_templates, + sim_object.reverse_multiscale_mapping, + ) + return nothing +end + """ status_from_template(d::Dict{Symbol,Any}) @@ -314,7 +394,11 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion # before we keep going (organ_with_vector, no_vectors_found) = (check_statuses_contain_no_remaining_vectors(mapping)) if !no_vectors_found - @assert false "Error : Mapping status at $organ_with_vector level contains a vector. If this was intentional, call the function generate_models_from_status_vectors on your mapping before calling run!. And bear in mind this is not meant for production. If this wasn't intentional, then it's likely an issue on the mapping definition, or an unusual model." + error( + "Error : Mapping status at $organ_with_vector level contains a vector. ", + "If this was intentional, call the function generate_models_from_status_vectors on your mapping before calling run!. ", + "And bear in mind this is not meant for production. If this wasn't intentional, then it's likely an issue on the mapping definition, or an unusual model." + ) end models = Dict(first(m) => parse_models(get_models(last(m))) for m in mapping) @@ -328,14 +412,15 @@ function init_simulation(mtg, mapping; nsteps=1, outputs=nothing, type_promotion scale_reachability = _scale_reachability_from_mtg(mtg) _infer_timestep_hints!(model_specs) - ignored_same_rate_hard_children = _same_rate_hard_dependency_children(model_specs, soft_dep_graphs_roots) - active_processes_by_scale = _active_processes_for_inference(model_specs, ignored_same_rate_hard_children) + validate_hard_dependency_timestep_consistency(model_specs, soft_dep_graphs_roots) + ignored_hard_children = _hard_dependency_children(soft_dep_graphs_roots) + active_processes_by_scale = _active_processes_for_inference(model_specs, ignored_hard_children) infer_model_specs_configuration!( model_specs; scale_reachability=scale_reachability, active_processes_by_scale=active_processes_by_scale ) - validate_model_specs_configuration(model_specs) + validate_model_specs_configuration(model_specs; ignored_processes_by_scale=ignored_hard_children) # Get the status of each node by node type, pre-initialised considering multi-scale variables: statuses, status_templates, reverse_multiscale_mapping, vars_need_init = diff --git a/src/mtg/mapping/compute_mapping.jl b/src/mtg/mapping/compute_mapping.jl index a6ccabccf..531e76d76 100644 --- a/src/mtg/mapping/compute_mapping.jl +++ b/src/mtg/mapping/compute_mapping.jl @@ -78,9 +78,6 @@ function variables_outputs_from_other_scale(mapped_vars) isnothing(orgs) && continue orgs_iterable = isa(orgs, Symbol) ? Symbol[orgs] : Symbol[orgs...] - filter!(o -> o != Symbol(""), orgs_iterable) - length(orgs_iterable) == 0 && continue # This can happen when we use a PreviousTimeStep alone - for o in orgs_iterable # The MappedVar can only have one value for the default, because it comes from the computing scale (the source scale): var_default_value = mapped_default(val) @@ -89,10 +86,14 @@ function variables_outputs_from_other_scale(mapped_vars) # The variable is written to several organs, the default value must be a vector: if isa(var_default_value, AbstractVector) # Mapping into a vector of organs, the default value must be a vector: - @assert length(var_default_value) == 1 "The variable `$(mapped_variable(val))` is an output variable computed by scale `$organ` and written to organs at scale `$(join(mapped_organ(val), ", "))`, " * - "but the default value coming from `$organ` is not of length 1: $(var_default_value). " * - "Make sure the model that computes this variable at scale `$organ` has a vector of values of length 1 as " * - "default outputs for variable `$(mapped_variable(val))`." + if length(var_default_value) != 1 + error( + "The variable `$(mapped_variable(val))` is an output variable computed by scale `$organ` and written to organs at scale `$(join(mapped_organ(val), ", "))`, " * + "but the default value coming from `$organ` is not of length 1: $(var_default_value). " * + "Make sure the model that computes this variable at scale `$organ` has a vector of values of length 1 as " * + "default outputs for variable `$(mapped_variable(val))`." + ) + end var_default_value = var_default_value[1] else error( @@ -104,11 +105,15 @@ function variables_outputs_from_other_scale(mapped_vars) end else # The variables is mapped to a single organ, the default value must be a scalar: - @assert !isa(var_default_value, AbstractVector) "The variable `$(mapped_variable(val))` is an output variable computed by scale `$organ` and written to organ at scale `$o`, " * - "but the default value coming from `$organ` is a vector: $(var_default_value). " * - "Make sure the model that computes this variable at scale `$organ` has a scalar value as " * - "default outputs for variable `$(mapped_variable(val))` (*e.g.* $(var_default_value[1])), or update your mapping to map the organ as a vector: " * - """`$(mapped_variable(val)) => ["$o"]`.""" + if isa(var_default_value, AbstractVector) + error( + "The variable `$(mapped_variable(val))` is an output variable computed by scale `$organ` and written to organ at scale `$o`, " * + "but the default value coming from `$organ` is a vector: $(var_default_value). " * + "Make sure the model that computes this variable at scale `$organ` has a scalar value as " * + "default outputs for variable `$(mapped_variable(val))` (*e.g.* $(var_default_value[1])), or update your mapping to map the organ as a vector: " * + """`$(mapped_variable(val)) => ["$o"]`.""" + ) + end end # We make a MappedVar object to declare the variable as an input of this scale: # mapped_var = MappedVar( @@ -165,12 +170,15 @@ function transform_single_node_mapped_variables_as_self_node_output!(mapped_vars for (var, mapped_var) in pairs(vars) # e.g. var = :carbon_biomass; mapped_var = vars[var] if isa(mapped_var, MappedVar{SingleNodeMapping}) source_organ = mapped_organ(mapped_var) - source_organ == Symbol("") && continue # We skip the variables that are mapped to themselves (e.g. [PreviousTimeStep(:variable_name)], or just renaming a variable) - @assert source_organ != organ "Variable `$var` is mapped to its own scale in organ $organ. This is not allowed." + if source_organ == organ + error("Variable `$var` is mapped to its own scale in organ $organ. This is not allowed.") + end - @assert haskey(mapped_vars[:outputs], source_organ) "Scale $source_organ not found in the mapping, but mapped to the $organ scale." - @assert haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * - "scale `$organ`, but is not computed by any model at `$source_organ` scale." + haskey(mapped_vars[:outputs], source_organ) || error("Scale $source_organ not found in the mapping, but mapped to the $organ scale.") + haskey(mapped_vars[:outputs][source_organ], source_variable(mapped_var)) || error( + "The variable `$(source_variable(mapped_var))` is mapped from scale `$source_organ` to " * + "scale `$organ`, but is not computed by any model at `$source_organ` scale." + ) # If the source variable was already defined as a `MappedVar{SelfNodeMapping}` by another scale, we skip it: isa(mapped_vars[:outputs][source_organ][source_variable(mapped_var)], MappedVar{SelfNodeMapping}) && continue @@ -211,10 +219,10 @@ function get_multiscale_default_value(mapped_vars, val, mapping_stacktrace=[], l if isa(val, MappedVar) && !isa(val, MappedVar{SelfNodeMapping}) # If the value is a MappedVar, we must find the default value of the variable it is mapping into. # Except if it is mapping to itself, in which case we return the value as is. + _is_same_scale_mapping(val) && return mapped_default(val) level += 1 # Find the default value of the variable from the scale it is mapping into (upper scale): m_organ = mapped_organ(val) - m_organ == Symbol("") && return mapped_default(val) # We skip the variables that are mapped to themselves (e.g. [PreviousTimeStep(:variable_name)], or just renaming a variable) if isa(m_organ, Symbol) m_organ = [m_organ] @@ -258,7 +266,7 @@ function default_variables_from_mapping(mapped_vars, verbose=true) mapped_vars_mutable = Dict{Symbol,Dict{Symbol,Any}}(k => Dict(pairs(v)) for (k, v) in mapped_vars) for (organ, vars) in mapped_vars # organ = :Leaf; vars = mapped_vars[organ] for (var, val) in pairs(vars) # var = :carbon_biomass; val = getproperty(vars,var) - if isa(val, MappedVar) && !isa(val, MappedVar{SelfNodeMapping}) && mapped_organ(val) != Symbol("") + if isa(val, MappedVar) && !isa(val, MappedVar{SelfNodeMapping}) && !_is_same_scale_mapping(val) mapping_stacktrace = Any[(mapped_organ=organ, mapped_variable=var, mapped_value=mapped_default(mapped_vars[organ][var]), level=1)] default_value = get_multiscale_default_value(mapped_vars, val, mapping_stacktrace) mapped_vars_mutable[organ][var] = MappedVar(source_organs(val), mapped_variable(val), source_variable(val), default_value) @@ -299,7 +307,6 @@ function convert_reference_values!(mapped_vars::Dict{Symbol,Dict{Symbol,Any}}) for (k, v) in vars # e.g.: k = :aPPFD_larger_scale; v = vars[k] if isa(v, MappedVar{SelfNodeMapping}) || isa(v, MappedVar{SingleNodeMapping}) mapped_org = isa(v, MappedVar{SelfNodeMapping}) ? organ : mapped_organ(v) - mapped_org == Symbol("") && continue key = mapped_org => source_variable(v) # First time we encounter this variable as a MappedVar, we create its value into the dict_mapped_vars Dict: @@ -339,7 +346,7 @@ function convert_reference_values!(mapped_vars::Dict{Symbol,Dict{Symbol,Any}}) # Third pass: getting the same reference for the variables that are mapped at the same scale to another variable (changing its name): for (organ, vars) in mapped_vars # e.g.: organ = :Plant; vars = mapped_vars[organ] for (k, v) in vars # e.g.: k = :carbon_allocation; v = vars[k] - if isa(v, MappedVar) && mapped_organ(v) == Symbol("") + if _is_same_scale_mapping(v) mapped_var = mapped_variable(v) isa(mapped_var, PreviousTimeStep) && (mapped_var = mapped_var.variable) if mapped_var == source_variable(v) diff --git a/src/mtg/mapping/getters.jl b/src/mtg/mapping/getters.jl index c9512c7d2..1b9b4a2b0 100644 --- a/src/mtg/mapping/getters.jl +++ b/src/mtg/mapping/getters.jl @@ -102,7 +102,9 @@ See [`get_models`](@ref) for examples. """ function get_status(m) st = Status[i for i in m if isa(i, Status)] - @assert length(st) <= 1 "Only one status can be provided for each organ type." + if length(st) > 1 + error("Only one status can be provided for each organ type.") + end length(st) == 0 && return nothing return first(st) end diff --git a/src/mtg/mapping/mapping.jl b/src/mtg/mapping/mapping.jl index 51b34efd4..c8404ad4d 100755 --- a/src/mtg/mapping/mapping.jl +++ b/src/mtg/mapping/mapping.jl @@ -1,21 +1,6 @@ -""" - AbstractNodeMapping - -Abstract type for the type of node mapping, *e.g.* single node mapping or multiple node mapping. -""" -abstract type AbstractNodeMapping end - -@noinline function _warn_string_scale(context::Symbol) - Base.depwarn( - "String scale names are deprecated and will be removed in a future release. Use Symbol scales, e.g. `:Leaf` instead of `\"Leaf\"`.", - context - ) -end - _normalize_scale(scale::Symbol; warn::Bool=false, context::Symbol=:PlantSimEngine) = scale -function _normalize_scale(scale::AbstractString; warn::Bool=true, context::Symbol=:PlantSimEngine) - warn && _warn_string_scale(context) - return Symbol(scale) +function _normalize_scale(scale; warn::Bool=false, context::Symbol=:PlantSimEngine) + error("Scale names must be `Symbol`s, got `$(typeof(scale))` for `$(repr(scale))` in `$context`.") end """ @@ -28,9 +13,6 @@ struct SingleNodeMapping <: AbstractNodeMapping scale::Symbol end -SingleNodeMapping(scale::Union{Symbol,AbstractString}) = - SingleNodeMapping(_normalize_scale(scale; warn=scale isa AbstractString, context=:SingleNodeMapping)) - """ SelfNodeMapping() @@ -52,11 +34,9 @@ struct MultiNodeMapping <: AbstractNodeMapping scale::Vector{Symbol} end -MultiNodeMapping(scale::Union{Symbol,AbstractString}) = MultiNodeMapping([scale]) -function MultiNodeMapping(scale::AbstractVector{<:Union{Symbol,AbstractString}}) - normalized = Symbol[ - _normalize_scale(s; warn=s isa AbstractString, context=:MultiNodeMapping) for s in scale - ] +MultiNodeMapping(scale::Symbol) = MultiNodeMapping([scale]) +function MultiNodeMapping(scale::AbstractVector{<:Symbol}) + normalized = Symbol[_normalize_scale(s; context=:MultiNodeMapping) for s in scale] return MultiNodeMapping(normalized) end @@ -93,15 +73,20 @@ source_organs(m::MappedVar) = m.source_organ source_organs(m::MappedVar{O,V1,V2,T}) where {O<:AbstractNodeMapping,V1,V2,T} = nothing mapped_organ(m::MappedVar{O,V1,V2,T}) where {O,V1,V2,T} = source_organs(m).scale mapped_organ(m::MappedVar{O,V1,V2,T}) where {O<:SelfNodeMapping,V1,V2,T} = nothing +mapped_organ(m::MappedVar{O,V1,V2,T}) where {O<:SameScale,V1,V2,T} = nothing mapped_organ_type(m::MappedVar{O,V1,V2,T}) where {O<:AbstractNodeMapping,V1,V2,T} = O source_variable(m::MappedVar) = m.source_variable function source_variable(m::MappedVar{O,V1,V2,T}, organ) where {O<:SingleNodeMapping,V1,V2<:Symbol,T} - @assert organ == mapped_organ(m) "Organ $organ not found in the mapping of the variable $(mapped_variable(m))." + if organ != mapped_organ(m) + error("Organ $organ not found in the mapping of the variable $(mapped_variable(m)).") + end m.source_variable end function source_variable(m::MappedVar{O,V1,V2,T}, organ) where {O<:MultiNodeMapping,V1,V2<:Vector{Symbol},T} - @assert organ in mapped_organ(m) "Organ $organ not found in the mapping of the variable $(mapped_variable(m))." + if !(organ in mapped_organ(m)) + error("Organ $organ not found in the mapping of the variable $(mapped_variable(m)).") + end m.source_variable[findfirst(o -> o == organ, mapped_organ(m))] end @@ -109,12 +94,19 @@ mapped_default(m::MappedVar) = m.source_default mapped_default(m::MappedVar{O,V1,V2,T}, organ) where {O<:MultiNodeMapping,V1,V2<:Vector{Symbol},T} = m.source_default[findfirst(o -> o == organ, mapped_organ(m))] mapped_default(m) = m # For any variable that is not a MappedVar, we return it as is +_is_same_scale_mapping(m::MappedVar) = source_organs(m) isa SameScale +_is_same_scale_mapping(::Any) = false + # This defines the type of mapping setup: either single or multiscale. Used to dispatch methods for e.g. `dep` or `to_initialize`. abstract type AbstractScaleSetup end struct MultiScale <: AbstractScaleSetup end struct SingleScale <: AbstractScaleSetup end +const ModelRateDeclarations = Dict{Symbol,Any} +const ReverseMappingTarget = Union{Symbol,PreviousTimeStep} +const ReverseMultiscaleMapping = Dict{Symbol,Dict{Symbol,Dict{Symbol,ReverseMappingTarget}}} + """ ModelMappingInfo @@ -128,7 +120,7 @@ struct ModelMappingInfo scales::Vector{Symbol} models_per_scale::Dict{Symbol,Int} processes_per_scale::Dict{Symbol,Vector{Symbol}} - declared_rates::Dict{Symbol,Any} + declared_rates::ModelRateDeclarations vars_need_init::Any model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}} recommendations::Vector{String} @@ -142,7 +134,7 @@ function _empty_model_mapping_info() Symbol[], Dict{Symbol,Int}(), Dict{Symbol,Vector{Symbol}}(), - Dict{Symbol,Any}(), + ModelRateDeclarations(), NamedTuple(), Dict{Symbol,Dict{Symbol,ModelSpec}}(), String[], @@ -171,7 +163,7 @@ configuration errors: The type behaves like a read-only dictionary keyed by scale name (`Symbol`). Use `Dict(mapping)` to recover a plain dictionary. """ -struct ModelMapping{S<:AbstractScaleSetup,D} <: AbstractDict{Symbol,Tuple} where {D<:Union{Dict{Symbol,Tuple},ModelList}} +struct ModelMapping{S<:AbstractScaleSetup,D} <: AbstractDict{Symbol,Tuple} where {D<:Union{Dict{Symbol,Tuple},SingleScaleModelSet}} data::D info::ModelMappingInfo type_promotion::Union{Nothing,Dict} @@ -303,20 +295,14 @@ Base.keys(::ModelMapping{SingleScale}) = (:Default,) Base.values(mapping::ModelMapping{SingleScale}) = ((values(mapping.data.models)..., status(mapping.data)),) Base.pairs(mapping::ModelMapping{SingleScale}) = (:Default => (values(mapping.data.models)..., status(mapping.data)),) Base.getindex(mapping::ModelMapping, key::Symbol) = mapping.data[key] -function Base.getindex(mapping::ModelMapping, key::AbstractString) - sym = _normalize_scale(key; warn=true, context=:ModelMapping) - return mapping.data[sym] -end function Base.getindex(mapping::ModelMapping{SingleScale}, key::Symbol) if key == :Default return (values(mapping.data.models)..., status(mapping.data)) end return getindex(mapping.data, key) end -Base.getindex(mapping::ModelMapping{SingleScale}, key::AbstractString) = getindex(mapping, _normalize_scale(key; warn=true, context=:ModelMapping)) Base.getindex(mapping::ModelMapping{SingleScale}, key::Integer) = getindex(mapping.data, key) Base.haskey(mapping::ModelMapping, key::Symbol) = haskey(mapping.data, key) -Base.haskey(mapping::ModelMapping, key::AbstractString) = haskey(mapping.data, _normalize_scale(key; warn=true, context=:ModelMapping)) Base.eltype(::Type{ModelMapping}) = Pair{Symbol,Tuple} Base.copy(mapping::ModelMapping{MultiScale}) = _build_model_mapping(MultiScale, copy(mapping.data); validated=mapping.info.validated, type_promotion=deepcopy(mapping.type_promotion)) Base.copy(mapping::ModelMapping{SingleScale}) = _build_model_mapping(SingleScale, copy(mapping.data); validated=mapping.info.validated, type_promotion=deepcopy(mapping.type_promotion)) @@ -359,7 +345,7 @@ end Convenience constructors for [`ModelMapping`](@ref): - pass `scale => models` pairs directly (dict-like syntax), -- or pass models/processes directly for a single scale (old `ModelList` syntax). +- or pass models/processes directly for a single scale. `type_promotion` may be a dictionary of source type to target type, for example `Dict(Float64 => Float32)`. In single-scale mappings it is applied when the @@ -368,33 +354,34 @@ mapping and applied when a `GraphSimulation` is initialized. """ function ModelMapping( args...; - scale::Union{Symbol,AbstractString}=:Default, + scale::Symbol=:Default, status=nothing, type_promotion::Union{Nothing,Dict}=nothing, check::Bool=true, processes... ) isempty(args) && isempty(processes) && error( - "No mapping or model was provided. Use `ModelMapping(\"Scale\" => models)` or pass models directly." + "No mapping or model was provided. Use `ModelMapping(:Scale => models)` or pass models directly." ) - - # Backwards compatibility: allow dict-like construction for type promotion maps, - # e.g. `ModelMapping(Float64 => Float32)`. - if !isempty(args) && all(arg -> arg isa Pair && !(first(arg) isa Union{AbstractString,Symbol}), args) - return Dict(args) - end + any(arg -> arg isa Pair && first(arg) isa AbstractString, args) && error( + "String scale names are removed. Use Symbol scales, e.g. `:Leaf` instead of `\"Leaf\"`." + ) + !isempty(args) && all(arg -> arg isa Pair && !(first(arg) isa Symbol), args) && error( + "`ModelMapping(Float64 => Float32)` type-promotion shorthand is removed. ", + "Use `Dict(Float64 => Float32)` as the `type_promotion` value." + ) if _all_scale_pairs(args) isempty(processes) || error( "Cannot mix scale-level pairs with process keyword arguments. ", - "Use either `\"Scale\" => models` pairs, or single-scale process/model arguments." + "Use either `:Scale => models` pairs, or single-scale process/model arguments." ) isnothing(status) || error( "`status` cannot be used with scale-level pair syntax. ", "Provide statuses inside each scale mapping instead." ) raw_mapping = Dict{Symbol,Any}( - _normalize_scale(first(pair); warn=first(pair) isa AbstractString, context=:ModelMapping) => last(pair) + _normalize_scale(first(pair); context=:ModelMapping) => last(pair) for pair in args ) return ModelMapping{MultiScale}(raw_mapping; check=check, type_promotion=type_promotion) @@ -419,12 +406,12 @@ function ModelMapping( return _build_model_mapping( SingleScale, - ModelList(flat_args...; status=status, type_promotion=type_promotion, variables_check=check, processes...); + SingleScaleModelSet(flat_args...; status=status, type_promotion=type_promotion, variables_check=check, processes...); validated=check, type_promotion=type_promotion ) - #TODO: Use the following when we merge the ModelList and ModelMapping paths (create a fake scale): + #TODO: Use the following when we merge the single-scale and multiscale mapping paths (create a fake scale): single_scale_models = _single_scale_mapping_entries(args, processes, status) # return ModelMapping{SingleScale}(Dict(_normalize_scale(scale) => single_scale_models), check=check) end @@ -456,11 +443,11 @@ type_promotion(::Any) = nothing _type_promotion(mapping) = type_promotion(mapping) function _all_scale_pairs(args) - !isempty(args) && all(arg -> arg isa Pair && first(arg) isa Union{AbstractString,Symbol}, args) + !isempty(args) && all(arg -> arg isa Pair && first(arg) isa Symbol, args) end function _contains_scale_like_pair(args) - any(arg -> arg isa Pair && first(arg) isa Union{AbstractString,Symbol}, args) + any(arg -> arg isa Pair && first(arg) isa Symbol, args) end function _single_scale_mapping_entries(args, processes, status) @@ -492,13 +479,13 @@ function _normalize_multiscale_mapping(mapping::AbstractDict) isempty(mapping) && error("ModelMapping cannot be empty. Provide at least one scale with models.") normalized = Dict{Symbol,Tuple}() for (scale, scale_mapping) in pairs(mapping) - scale_name = _normalize_scale(scale; warn=scale isa AbstractString, context=:ModelMapping) + scale_name = _normalize_scale(scale; context=:ModelMapping) normalized[scale_name] = _normalize_scale_mapping(scale_name, scale_mapping) end return normalized end -function _normalize_scale_mapping(scale::Symbol, scale_mapping::ModelList) +function _normalize_scale_mapping(scale::Symbol, scale_mapping::SingleScaleModelSet) return _normalize_scale_mapping(scale, (values(scale_mapping.models)..., status(scale_mapping))) end @@ -513,14 +500,14 @@ end function _normalize_scale_mapping(scale::Symbol, scale_mapping::Tuple) normalized_items = Any[] for item in scale_mapping - if item isa ModelList + if item isa SingleScaleModelSet append!(normalized_items, values(item.models)) push!(normalized_items, status(item)) elseif item isa Union{AbstractModel,MultiScaleModel,ModelSpec,Status} push!(normalized_items, item) else error( - "Invalid mapping entry at scale `$scale`: expected models/ModelSpec, Status, or ModelList, got $(typeof(item))." + "Invalid mapping entry at scale `$scale`: expected models, ModelSpec, Status, or single-scale ModelMapping, got $(typeof(item))." ) end end @@ -529,7 +516,7 @@ end function _normalize_scale_mapping(scale::Symbol, scale_mapping) error( - "Invalid mapping entry at scale `$scale`: expected a model/ModelSpec, tuple of models/Status, or ModelList, got $(typeof(scale_mapping))." + "Invalid mapping entry at scale `$scale`: expected a model, ModelSpec, tuple of models/Status, or single-scale ModelMapping, got $(typeof(scale_mapping))." ) end @@ -627,19 +614,15 @@ _mapping_item_mapped_variables(::Any) = Pair{Symbol,Symbol}[] _mapping_item_model(item::ModelSpec) = model_(item) _mapping_item_model(item::MultiScaleModel) = model_(item) -function _as_mapping_scale(source_scale::AbstractString) - isempty(source_scale) && return nothing - return _normalize_scale(source_scale; warn=true, context=:ModelMapping) -end - function _as_mapping_scale(source_scale::Symbol) - source_scale === Symbol("") && return nothing - return _normalize_scale(source_scale; warn=false, context=:ModelMapping) + source_scale === Symbol("") && error("`Symbol(\"\")` same-scale mappings are removed. Use `SameScale()` instead.") + return _normalize_scale(source_scale; context=:ModelMapping) end +_as_mapping_scale(::SameScale) = nothing -_as_mapping_sources(source::Pair{<:Union{AbstractString,Symbol},Symbol}) = +_as_mapping_sources(source::Pair{T,Symbol}) where {T<:Union{Symbol,SameScale}} = (_as_mapping_scale(first(source)) => last(source),) -_as_mapping_sources(source::AbstractVector{<:Pair{<:Union{AbstractString,Symbol},Symbol}}) = +_as_mapping_sources(source::AbstractVector{<:Pair}) = Tuple(_as_mapping_scale(first(item)) => last(item) for item in source) function _available_variables_by_scale(mapping::Dict{Symbol,Tuple}) @@ -678,7 +661,7 @@ function _available_variables_by_scale(mapping::Dict{Symbol,Tuple}) end function _declared_model_rates_by_scale(mapping::Dict{Symbol,Tuple}) - rates = Dict{Symbol,Any}() + rates = ModelRateDeclarations() for (scale, scale_mapping) in mapping declared_rates = unique(filter(!isnothing, map(model_rate, get_models(scale_mapping)))) if length(declared_rates) > 1 @@ -705,7 +688,7 @@ function _spec_declares_multirate(spec::ModelSpec) return false end -function _mapping_declares_multirate(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, declared_rates::Dict{Symbol,Any}) +function _mapping_declares_multirate(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, declared_rates::ModelRateDeclarations) any(!isnothing, values(declared_rates)) && return true for specs_at_scale in values(model_specs), spec in values(specs_at_scale) _spec_declares_multirate(spec) && return true @@ -746,14 +729,14 @@ function _build_model_mapping_recommendations( return recommendations end -function _build_model_mapping_info(::Type{SingleScale}, mapping::ModelList; validated::Bool) +function _build_model_mapping_info(::Type{SingleScale}, mapping::SingleScaleModelSet; validated::Bool) specs = Dict( :Default => Dict{Symbol,ModelSpec}( process(model) => as_model_spec(model) for model in values(mapping.models) ) ) - declared_rates = Dict{Symbol,Any}(:Default => nothing) + declared_rates = ModelRateDeclarations(:Default => nothing) vars_need_init = try to_initialize(mapping) catch @@ -782,7 +765,7 @@ function _build_model_mapping_info(::Type{MultiScale}, mapping::Dict{Symbol,Tupl declared_rates = try _declared_model_rates_by_scale(mapping) catch - Dict{Symbol,Any}(scale => nothing for scale in scales) + ModelRateDeclarations(scale => nothing for scale in scales) end model_specs = try _parse_model_specs_from_mapping(mapping) diff --git a/src/mtg/mapping/model_generation_from_status_vectors.jl b/src/mtg/mapping/model_generation_from_status_vectors.jl index 76cb22901..e0c3cad47 100644 --- a/src/mtg/mapping/model_generation_from_status_vectors.jl +++ b/src/mtg/mapping/model_generation_from_status_vectors.jl @@ -66,7 +66,6 @@ PlantSimEngine.TimeStepDependencyTrait(::Type{<:GeneratedStatusVectorModel}) = P function replace_mapping_status_vectors_with_generated_models(mapping_with_vectors_in_status, timestep_model_organ_level, nsteps) timestep_model_organ_level = _normalize_scale( timestep_model_organ_level; - warn=timestep_model_organ_level isa AbstractString, context=:ModelMapping ) @@ -78,7 +77,7 @@ function replace_mapping_status_vectors_with_generated_models(mapping_with_vecto # we are now certain a model will be generated, and that the timestep models need to be inserted mapping = Dict( - _normalize_scale(organ; warn=organ isa AbstractString, context=:ModelMapping) => models + _normalize_scale(organ; context=:ModelMapping) => models for (organ, models) in mapping_with_vectors_in_status ) for (organ,models) in mapping @@ -129,8 +128,8 @@ function replace_mapping_status_vectors_with_generated_models(mapping_with_vecto end function generate_model_from_status_vector_variable(mapping, timestep_scale, status, organ, nsteps) - timestep_scale = _normalize_scale(timestep_scale; warn=timestep_scale isa AbstractString, context=:ModelMapping) - organ = _normalize_scale(organ; warn=organ isa AbstractString, context=:ModelMapping) + timestep_scale = _normalize_scale(timestep_scale; context=:ModelMapping) + organ = _normalize_scale(organ; context=:ModelMapping) # Ah, another point that remains to be seen is that those CSV.SentinelArrays.ChainedVector obtained from the meteo file isn't an AbstractVector # meaning currently we won't generate models from them unless the conversion is made before that @@ -144,9 +143,13 @@ function generate_model_from_status_vector_variable(mapping, timestep_scale, sta for symbol in keys(status) value = getproperty(status, symbol) if isa(value, AbstractVector) - @assert length(value) > 0 "Error during generation of models from vector values provided at the $organ-level status : provided $symbol vector is empty" + if length(value) == 0 + error("Error during generation of models from vector values provided at the $organ-level status : provided $symbol vector is empty") + end # TODO : Might need to fiddle with timesteps here in the future in case of varying timestep models - @assert nsteps == length(value) "Error during generation of models from vector values provided at the $organ-level status : provided $symbol vector length doesn't match the expected # of timesteps" + if nsteps != length(value) + error("Error during generation of models from vector values provided at the $organ-level status : provided $symbol vector length doesn't match the expected # of timesteps") + end process_name = Symbol(lowercase(string(symbol) * bytes2hex(sha1(repr(value))))) model = GeneratedStatusVectorModel(process_name, symbol, value) @@ -172,13 +175,15 @@ function generate_model_from_status_vector_variable(mapping, timestep_scale, sta new_status = Status(NamedTuple{Tuple(new_status_names)}(Tuple(new_status_values))) generated_models_tuple = Tuple(generated_models) - @assert length(status) == length(new_status) + length(generated_models_tuple) "Error during generation of models from vector values provided at the $organ-level status" + if length(status) != length(new_status) + length(generated_models_tuple) + error("Error during generation of models from vector values provided at the $organ-level status") + end return new_status, generated_models_tuple end # This is a helper function only for testing purposes. -function modellist_to_mapping(modellist_original::ModelList, modellist_status; nsteps=nothing, outputs=nothing) +function modellist_to_mapping(modellist_original::SingleScaleModelSet, modellist_status; nsteps=nothing, outputs=nothing) modellist = Base.copy(modellist_original, modellist_original.status) @@ -273,5 +278,5 @@ function check_statuses_contain_no_remaining_vectors(mapping) end end end - return (Symbol(""), true) + return (nothing, true) end diff --git a/src/mtg/mapping/reverse_mapping.jl b/src/mtg/mapping/reverse_mapping.jl index aafca47a5..ccf6255e9 100644 --- a/src/mtg/mapping/reverse_mapping.jl +++ b/src/mtg/mapping/reverse_mapping.jl @@ -77,7 +77,7 @@ function reverse_mapping(mapping::AbstractDict{Symbol,T}; all=true) where {T<:An end function reverse_mapping(mapped_vars::Dict{Symbol,Dict{Symbol,Any}}; all=true) - reverse_multiscale_mapping = Dict{Symbol,Dict{Symbol,Dict{Symbol,Any}}}(org => Dict{Symbol,Dict{Symbol,Any}}() for org in keys(mapped_vars)) + reverse_multiscale_mapping = ReverseMultiscaleMapping(org => Dict{Symbol,Dict{Symbol,ReverseMappingTarget}}() for org in keys(mapped_vars)) for (organ, vars) in mapped_vars # e.g.: organ = :Plant; vars = mapped_vars[organ] for (var, val) in vars # e.g. var = :Rm_organs; val = vars[var] if isa(val, MappedVar) && !isa(val, MappedVar{SelfNodeMapping}) && (all || !isa(val, MappedVar{SingleNodeMapping})) @@ -90,16 +90,15 @@ function reverse_mapping(mapped_vars::Dict{Symbol,Dict{Symbol,Any}}; all=true) if mapped_orgs isa Symbol mapped_orgs = [mapped_orgs] elseif mapped_orgs isa AbstractString - mapped_orgs = [_normalize_scale(mapped_orgs; warn=true, context=:ModelMapping)] + error("String scale names are removed. Use Symbol scales, e.g. `:Leaf` instead of `\"Leaf\"`.") end for mapped_o in mapped_orgs # e.g.: mapped_o = :Leaf - mapped_o == Symbol("") && continue # if !haskey(reverse_multiscale_mapping, mapped_o) # reverse_multiscale_mapping[mapped_o] = Dict{Symbol,Vector{MappedVar}}() # end if !haskey(reverse_multiscale_mapping[mapped_o], organ) - reverse_multiscale_mapping[mapped_o][organ] = Dict{Symbol,Any}(source_variable(val, mapped_o) => mapped_variable(val)) + reverse_multiscale_mapping[mapped_o][organ] = Dict{Symbol,ReverseMappingTarget}(source_variable(val, mapped_o) => mapped_variable(val)) end push!(reverse_multiscale_mapping[mapped_o][organ], source_variable(val, mapped_o) => mapped_variable(val)) end diff --git a/src/mtg/model_spec_inference.jl b/src/mtg/model_spec_inference.jl index 00dfd6ba3..94814876f 100644 --- a/src/mtg/model_spec_inference.jl +++ b/src/mtg/model_spec_inference.jl @@ -243,11 +243,12 @@ function _same_timestep_signature(sig_a, sig_b) end function _hard_dep_same_rate_as_parent(model_specs, parent_scale::Symbol, parent_process::Symbol, child_scale::Symbol, child_process::Symbol) - parent_scale == child_scale || return false parent_specs = get(model_specs, parent_scale, nothing) + child_specs = get(model_specs, child_scale, nothing) isnothing(parent_specs) && return false + isnothing(child_specs) && return false parent_spec = get(parent_specs, parent_process, nothing) - child_spec = get(parent_specs, child_process, nothing) + child_spec = get(child_specs, child_process, nothing) isnothing(parent_spec) && return false isnothing(child_spec) && return false @@ -280,6 +281,41 @@ function _collect_same_rate_hard_dependency_children!( return nothing end +function _collect_hard_dependency_children!( + ignored_processes_by_scale::Dict{Symbol,Set{Symbol}}, + child::HardDependencyNode +) + push!(get!(ignored_processes_by_scale, child.scale, Set{Symbol}()), child.process) + for nested in child.children + _collect_hard_dependency_children!(ignored_processes_by_scale, nested) + end + return nothing +end + +function _validate_hard_dependency_timestep_consistency!( + model_specs, + parent_scale::Symbol, + parent_process::Symbol, + child::HardDependencyNode +) + _hard_dep_same_rate_as_parent(model_specs, parent_scale, parent_process, child.scale, child.process) || error( + "Hard dependency `$(child.process)` at scale `$(child.scale)` has a different timestep than parent process ", + "`$(parent_process)` at scale `$(parent_scale)`. Hard dependencies are called manually by their parent, ", + "so their `ModelSpec` timestep cannot be scheduled independently. Align the timesteps, or make the child a soft dependency." + ) + + for nested in child.children + _validate_hard_dependency_timestep_consistency!( + model_specs, + child.scale, + child.process, + nested + ) + end + + return nothing +end + function _soft_nodes_for_hard_dependency_analysis(dep_graph::DependencyGraph{Dict{Symbol,Any}}) nodes = SoftDependencyNode[] for (_, roots_at_scale) in pairs(dep_graph.roots) @@ -309,6 +345,33 @@ function _same_rate_hard_dependency_children(model_specs, dep_graph::DependencyG return ignored_processes_by_scale end +function _hard_dependency_children(dep_graph::DependencyGraph) + ignored_processes_by_scale = Dict{Symbol,Set{Symbol}}() + + for soft_node in _soft_nodes_for_hard_dependency_analysis(dep_graph) + for child in soft_node.hard_dependency + _collect_hard_dependency_children!(ignored_processes_by_scale, child) + end + end + + return ignored_processes_by_scale +end + +function validate_hard_dependency_timestep_consistency(model_specs, dep_graph::DependencyGraph) + for soft_node in _soft_nodes_for_hard_dependency_analysis(dep_graph) + for child in soft_node.hard_dependency + _validate_hard_dependency_timestep_consistency!( + model_specs, + soft_node.scale, + soft_node.process, + child + ) + end + end + + return nothing +end + function _active_processes_for_inference(model_specs, ignored_processes_by_scale::Dict{Symbol,Set{Symbol}}) active = Dict{Symbol,Set{Symbol}}() for (scale, specs_at_scale) in pairs(model_specs) @@ -368,6 +431,28 @@ function _default_policy_for_inferred_binding(model_specs, source_scale::Symbol, ) end +function _terminal_update_candidate(model_specs, candidates::Vector, var::Symbol) + length(candidates) > 1 || return nothing + + terminal_updates = NamedTuple[] + for candidate in candidates + candidate_spec = model_specs[candidate.scale][candidate.process] + var in _update_variables_for_spec(candidate_spec) || continue + has_later_update = any(candidates) do other + other.scale == candidate.scale || return false + other.process == candidate.process && return false + other.var == var || return false + other_spec = model_specs[other.scale][other.process] + var in _update_variables_for_spec(other_spec) || return false + return candidate.process in _update_after_for_var(other_spec, var) + end + has_later_update || push!(terminal_updates, candidate) + end + + length(terminal_updates) == 1 || return nothing + return only(terminal_updates) +end + function _mapped_source_scales_for_input(spec::ModelSpec, input_var::Symbol) mapped = mapped_variables_(spec) isempty(mapped) && return Set{Symbol}() @@ -379,14 +464,14 @@ function _mapped_source_scales_for_input(spec::ModelSpec, input_var::Symbol) mapped_input == input_var || continue rhs = last(mv) - if rhs isa Pair{Symbol,Symbol} + if rhs isa Pair src_scale = first(rhs) - src_scale == Symbol("") || push!(scales, src_scale) + src_scale isa SameScale || push!(scales, src_scale) elseif rhs isa AbstractVector for item in rhs - item isa Pair{Symbol,Symbol} || continue + item isa Pair || continue src_scale = first(item) - src_scale == Symbol("") || push!(scales, src_scale) + src_scale isa SameScale || push!(scales, src_scale) end end end @@ -411,18 +496,18 @@ function _mapped_sources_for_input(spec::ModelSpec, input_var::Symbol) mapped = mapped_variables_(spec) isempty(mapped) && return Pair{Symbol,Symbol}[] - sources = Pair{Symbol,Symbol}[] + sources = Pair{Union{Symbol,SameScale},Symbol}[] for mv in mapped mapped_input = first(mv) mapped_input = mapped_input isa PreviousTimeStep ? mapped_input.variable : mapped_input mapped_input == input_var || continue rhs = last(mv) - if rhs isa Pair{Symbol,Symbol} + if rhs isa Pair push!(sources, rhs) elseif rhs isa AbstractVector for item in rhs - item isa Pair{Symbol,Symbol} || continue + item isa Pair || continue push!(sources, item) end end @@ -445,7 +530,7 @@ function _infer_binding_from_multiscale_mapping( mapped_sources = _mapped_sources_for_input(spec, input_var) # Mapping exists but does not point to another scale (self/same-scale aliasing): # avoid generic same-name inference in that case. - filtered_sources = filter(s -> first(s) != Symbol(""), mapped_sources) + filtered_sources = filter(s -> !(first(s) isa SameScale), mapped_sources) isempty(filtered_sources) && return :skip # Multi-source mapping (e.g. vectors from several scales) cannot be represented @@ -457,7 +542,7 @@ function _infer_binding_from_multiscale_mapping( src_var = last(src) haskey(model_specs, src_scale) || return :skip - procs = Symbol[] + candidates = NamedTuple[] for (src_process, src_spec) in pairs(model_specs[src_scale]) if !isnothing(active_processes_by_scale) active = get(active_processes_by_scale, src_scale, Set{Symbol}()) @@ -465,11 +550,17 @@ function _infer_binding_from_multiscale_mapping( end src_var in keys(outputs_(model_(src_spec))) || continue _is_stream_only_output(src_spec, src_var) && continue - push!(procs, src_process) + push!(candidates, (scale=src_scale, process=src_process, var=src_var)) end - length(procs) == 1 || return :skip - src_process = only(procs) + if length(candidates) == 1 + candidate = only(candidates) + else + candidate = _terminal_update_candidate(model_specs, candidates, src_var) + isnothing(candidate) && return :skip + end + + src_process = candidate.process policy = _default_policy_for_inferred_binding(model_specs, src_scale, src_process, src_var) return (process=src_process, var=src_var, scale=src_scale, policy=policy) end @@ -496,6 +587,21 @@ function _infer_input_binding_for_var( policy = _default_policy_for_inferred_binding(model_specs, c.scale, c.process, c.var) return (process=c.process, var=c.var, scale=c.scale, policy=policy) elseif length(same_scale) > 1 + terminal_update = _terminal_update_candidate(model_specs, same_scale, input_var) + if !isnothing(terminal_update) + policy = _default_policy_for_inferred_binding( + model_specs, + terminal_update.scale, + terminal_update.process, + terminal_update.var, + ) + return ( + process=terminal_update.process, + var=terminal_update.var, + scale=terminal_update.scale, + policy=policy, + ) + end error( "Ambiguous inferred producer for input `$(input_var)` in process `$(process)` at scale `$(scale)`. ", "Multiple same-scale candidates were found: $(_format_candidate_list(same_scale)). ", @@ -685,18 +791,27 @@ For a `GraphSimulation`, this returns the already resolved model specs used by t function resolved_model_specs(mapping::AbstractDict; infer::Bool=true, validate::Bool=true) model_specs = Dict{Symbol,Dict{Symbol,ModelSpec}}() for (scale, declarations) in pairs(mapping) - scale_sym = if scale isa Symbol - scale - elseif scale isa AbstractString - _normalize_scale(scale; warn=true, context=:ModelSpec) - else - error("Scale keys in `resolved_model_specs(mapping)` must be `Symbol` (preferred) or `String`, got `$(typeof(scale))`.") - end + scale_sym = scale isa Symbol ? + scale : + error("Scale keys in `resolved_model_specs(mapping)` must be `Symbol`s, got `$(typeof(scale))`.") model_specs[scale_sym] = parse_model_specs(declarations) end infer && infer_model_specs_configuration!(model_specs) - validate && validate_model_specs_configuration(model_specs) + if validate + hard_dep_roots = try + first(hard_dependencies(mapping; verbose=false)) + catch + nothing + end + ignored_hard_children = if isnothing(hard_dep_roots) + Dict{Symbol,Set{Symbol}}() + else + validate_hard_dependency_timestep_consistency(model_specs, hard_dep_roots) + _hard_dependency_children(hard_dep_roots) + end + validate_model_specs_configuration(model_specs; ignored_processes_by_scale=ignored_hard_children) + end return model_specs end @@ -718,12 +833,20 @@ function _model_specs_rows(model_specs) scale=scale, process=process, model=typeof(model_(spec)), + application_name=application_name(spec), + applies_to=applies_to(spec), + value_inputs=value_inputs(spec), + model_calls=model_calls(spec), + environment=environment_config(spec), timestep=timestep(spec), timespec_default=timespec(model_(spec)), timestep_resolution=resolution, input_bindings=input_bindings(spec), meteo_bindings=meteo_bindings(spec), meteo_window=meteo_window(spec), + meteo_inputs=meteo_inputs_(spec), + meteo_outputs=meteo_outputs_(spec), + updates=updates(spec), )) end end @@ -740,10 +863,18 @@ Summary fields: - `scale` - `process` - `model` +- `application_name` +- `applies_to` +- `value_inputs` +- `model_calls` +- `environment` - `timestep` - `input_bindings` - `meteo_bindings` - `meteo_window` +- `meteo_inputs` +- `meteo_outputs` +- `updates` """ function explain_model_specs(target; io::IO=stdout, infer::Bool=true, validate::Bool=true) specs = target isa GraphSimulation ? resolved_model_specs(target) : resolved_model_specs(target; infer=infer, validate=validate) @@ -766,6 +897,14 @@ function explain_model_specs(target; io::IO=stdout, infer::Bool=true, validate:: input_bindings_desc = (row.input_bindings isa NamedTuple && isempty(keys(row.input_bindings))) ? "(none)" : _stringify_compact(row.input_bindings) meteo_bindings_desc = (row.meteo_bindings isa NamedTuple && isempty(keys(row.meteo_bindings))) ? "(none)" : _stringify_compact(row.meteo_bindings) meteo_window_desc = isnothing(row.meteo_window) ? "(default rolling)" : _stringify_compact(row.meteo_window) + meteo_inputs_desc = (row.meteo_inputs isa NamedTuple && isempty(keys(row.meteo_inputs))) ? "(none)" : _stringify_compact(row.meteo_inputs) + meteo_outputs_desc = (row.meteo_outputs isa NamedTuple && isempty(keys(row.meteo_outputs))) ? "(none)" : _stringify_compact(row.meteo_outputs) + updates_desc = isempty(row.updates) ? "(none)" : _stringify_compact(row.updates) + application_name_desc = isnothing(row.application_name) ? "(unnamed)" : string(row.application_name) + applies_to_desc = isnothing(row.applies_to) ? "(implicit legacy target)" : _stringify_compact(row.applies_to) + value_inputs_desc = (row.value_inputs isa NamedTuple && isempty(keys(row.value_inputs))) ? "(none)" : _stringify_compact(row.value_inputs) + model_calls_desc = (row.model_calls isa NamedTuple && isempty(keys(row.model_calls))) ? "(none)" : _stringify_compact(row.model_calls) + environment_desc = isnothing(row.environment) ? "(default)" : _stringify_compact(row.environment) println( io, " - ", @@ -774,14 +913,30 @@ function explain_model_specs(target; io::IO=stdout, infer::Bool=true, validate:: row.process, " [", row.model, - "]: timestep=", + "]: name=", + application_name_desc, + ", applies_to=", + applies_to_desc, + ", value_inputs=", + value_inputs_desc, + ", model_calls=", + model_calls_desc, + ", environment=", + environment_desc, + ", timestep=", timestep_desc, ", input_bindings=", input_bindings_desc, ", meteo_bindings=", meteo_bindings_desc, ", meteo_window=", - meteo_window_desc + meteo_window_desc, + ", meteo_inputs=", + meteo_inputs_desc, + ", meteo_outputs=", + meteo_outputs_desc, + ", updates=", + updates_desc ) end return rows diff --git a/src/mtg/model_spec_validation.jl b/src/mtg/model_spec_validation.jl index e45cbfd6f..c3cbc9192 100644 --- a/src/mtg/model_spec_validation.jl +++ b/src/mtg/model_spec_validation.jl @@ -179,9 +179,9 @@ function _validate_binding_target( ) isnothing(source_scale) && return nothing - src_scale = source_scale isa AbstractString ? - _normalize_scale(source_scale; warn=true, context=:ModelSpec) : - source_scale + src_scale = source_scale isa Symbol ? + source_scale : + error("Source scale for input `$(input_var)` in process `$(process)` must be a `Symbol`, got `$(typeof(source_scale))`.") haskey(model_specs, src_scale) || error( "Unknown source scale `$(src_scale)` for input `$(input_var)` in process `$(process)` at scale `$(scale)`." ) @@ -232,9 +232,9 @@ function _validate_input_binding( end if haskey(binding, :scale) - isnothing(binding.scale) || binding.scale isa Symbol || binding.scale isa AbstractString || error( + isnothing(binding.scale) || binding.scale isa Symbol || error( "Invalid input binding for input `$(input_var)` in process `$(process)` at scale `$(scale)`: ", - "`scale` must be a Symbol, String or `nothing`, got `$(typeof(binding.scale))`." + "`scale` must be a Symbol or `nothing`, got `$(typeof(binding.scale))`." ) source_scale = binding.scale end @@ -419,7 +419,10 @@ end Validate mapping-level `ModelSpec` configuration before simulation runtime starts. This catches invalid timestep declarations, input bindings and output routing early. """ -function validate_model_specs_configuration(model_specs) +function validate_model_specs_configuration( + model_specs; + ignored_processes_by_scale::Dict{Symbol,Set{Symbol}}=Dict{Symbol,Set{Symbol}}() +) known_processes = Set{Symbol}() for specs_at_scale in values(model_specs) union!(known_processes, keys(specs_at_scale)) @@ -435,6 +438,7 @@ function validate_model_specs_configuration(model_specs) _validate_output_routing_for_spec(scale, process, spec) end end + validate_update_dependencies(model_specs; ignored_processes_by_scale=ignored_processes_by_scale) return nothing end diff --git a/src/mtg/node_mapping_types.jl b/src/mtg/node_mapping_types.jl new file mode 100644 index 000000000..9ef0dd753 --- /dev/null +++ b/src/mtg/node_mapping_types.jl @@ -0,0 +1,15 @@ +""" + AbstractNodeMapping + +Abstract type for node mapping markers, such as single-node, multi-node, self, +or same-scale mappings. +""" +abstract type AbstractNodeMapping end + +""" + SameScale() + +Marker used in multiscale variable mappings when a variable is aliased or +renamed from another variable at the same scale. +""" +struct SameScale <: AbstractNodeMapping end diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 8230f534a..b1b040699 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -126,7 +126,9 @@ function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_ma else for i in keys(outs) # i = :Plant i isa Symbol || error("Output scale keys must be `Symbol`, got `$(typeof(i))` for key `$(repr(i))`.") - @assert isa(outs[i], Tuple{Vararg{Symbol}}) """Outputs for scale $i should be a tuple of symbols, *e.g.* `$i => (:a, :b)`, found `$i => $(outs[i])` instead.""" + if !isa(outs[i], Tuple{Vararg{Symbol}}) + error("""Outputs for scale $i should be a tuple of symbols, *e.g.* `$i => (:a, :b)`, found `$i => $(outs[i])` instead.""") + end outs_[i] = [outs[i]...] end end @@ -207,7 +209,9 @@ function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_ma end node_type = unique(node_types) - @assert length(node_type) == 1 "All plant graph nodes should have the same type, found $(unique(node_type))." + if length(node_type) != 1 + error("All plant graph nodes should have the same type, found $(unique(node_type)).") + end node_type = only(node_type) # I don't know if this function barrier is necessary @@ -317,7 +321,7 @@ function copy_tracked_outputs_into_vector!(outs_organ, i, statuses_organ, tracke return j end -function pre_allocate_outputs(m::ModelList, outs, nsteps; type_promotion=nothing, check=true) +function pre_allocate_outputs(m::SingleScaleModelSet, outs, nsteps; type_promotion=nothing, check=true) st, = flatten_status(status(m)) out_vars_all = convert_vars(st, type_promotion) diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 7c999d903..1c1d2fde6 100755 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -66,7 +66,7 @@ mapping = ModelMapping( to_initialize(mapping) ``` """ -function to_initialize(m::ModelList) +function to_initialize(m::SingleScaleModelSet) needed_variables = to_initialize(dep(m)) to_init = Dict{Symbol,Tuple}() for (process, vars) in needed_variables @@ -90,10 +90,11 @@ function to_initialize(m::DependencyGraph) for (key, value) in dependencies for (key_in, val_in) in pairs(value.inputs) if key_in ∉ outputs_all + input_default = NamedTuple{(key_in,)}((val_in,)) if haskey(needed_variables_process, key) - needed_variables_process[key] = merge(needed_variables_process[key], NamedTuple{(key_in,)}(val_in)) + needed_variables_process[key] = merge(needed_variables_process[key], input_default) else - push!(needed_variables_process, key => NamedTuple{(key_in,)}(val_in)) + push!(needed_variables_process, key => input_default) end end end @@ -245,7 +246,7 @@ function init_variables(model::T; verbose::Bool=true) where {T<:AbstractModel} return vars end -function init_variables(m::ModelList; verbose::Bool=true) +function init_variables(m::SingleScaleModelSet; verbose::Bool=true) init_variables(dep(m)) end @@ -300,7 +301,7 @@ models = ModelMapping( is_initialized(models) ``` """ -function is_initialized(m::T; verbose=true) where {T<:ModelList} +function is_initialized(m::T; verbose=true) where {T<:SingleScaleModelSet} var_names = to_initialize(m) if any([length(to_init) > 0 for (process, to_init) in pairs(var_names)]) diff --git a/src/processes/models_inputs_outputs.jl b/src/processes/models_inputs_outputs.jl index 02bac7eb1..ec8c4253c 100644 --- a/src/processes/models_inputs_outputs.jl +++ b/src/processes/models_inputs_outputs.jl @@ -59,6 +59,41 @@ or `Interpolate`) for each producer output. output_policy(model::AbstractModel) = output_policy(typeof(model)) output_policy(::Type{<:AbstractModel}) = NamedTuple() +""" + application_name(spec::ModelSpec) + +Optional stable name for one model application in the unified scene/object API. +""" +application_name(spec::ModelSpec) = spec.name + +""" + applies_to(spec::ModelSpec) + +Object selector where a model application runs in the unified scene/object API. +""" +applies_to(spec::ModelSpec) = spec.applies_to + +""" + value_inputs(spec::ModelSpec) + +Unified scene/object value-input bindings declared with `Inputs(...)`. +""" +value_inputs(spec::ModelSpec) = spec.inputs + +""" + model_calls(spec::ModelSpec) + +Unified scene/object manual call bindings declared with `Calls(...)`. +""" +model_calls(spec::ModelSpec) = spec.calls + +""" + environment_config(spec::ModelSpec) + +Optional scene/object environment configuration declared with `Environment(...)`. +""" +environment_config(spec::ModelSpec) = spec.environment + """ input_bindings(spec::ModelSpec) @@ -102,6 +137,14 @@ Default is `:global`. """ model_scope(spec::ModelSpec) = spec.scope +""" + updates(spec::ModelSpec) + +Scenario-level metadata for variables intentionally updated by this model after +another producer in the same mapping. +""" +updates(spec::ModelSpec) = spec.updates + """ meteo_bindings(spec::ModelSpec) @@ -122,6 +165,32 @@ Defaults to `nothing` (runtime falls back to `PlantMeteo.RollingWindow()` behavi """ meteo_window(spec::ModelSpec) = spec.meteo_window +""" + meteo_inputs(model::AbstractModel) + meteo_inputs_(model::AbstractModel) + +Meteorological/environment variables read directly by a model. + +This trait is separate from `inputs_` because meteorology may be constant, +table-backed, or produced by a microclimate domain. The default is empty. +""" +meteo_inputs(model::AbstractModel) = keys(meteo_inputs_(model)) +meteo_inputs(spec::ModelSpec) = keys(meteo_inputs_(spec)) +meteo_inputs_(model::AbstractModel) = NamedTuple() +meteo_inputs_(model::Missing) = NamedTuple() + +""" + meteo_outputs(model::AbstractModel) + meteo_outputs_(model::AbstractModel) + +Meteorological/environment variables produced by a model, for example local +microclimate variables computed over a voxel/octree domain. +""" +meteo_outputs(model::AbstractModel) = keys(meteo_outputs_(model)) +meteo_outputs(spec::ModelSpec) = keys(meteo_outputs_(spec)) +meteo_outputs_(model::AbstractModel) = NamedTuple() +meteo_outputs_(model::Missing) = NamedTuple() + """ inputs(mapping::ModelMapping) inputs(mapping::AbstractDict{Symbol,T}) diff --git a/src/run.jl b/src/run.jl index 99cbe6d15..cfe4a0b79 100644 --- a/src/run.jl +++ b/src/run.jl @@ -114,30 +114,16 @@ function adjust_weather_timesteps_to_given_length(desired_length, meteo) end -function _all_modellists_collection(object) - if isa(object, AbstractArray) - return all(x -> x isa ModelList || x isa ModelMapping{SingleScale}, object) - elseif isa(object, AbstractDict) - return all(x -> x isa ModelList || x isa ModelMapping{SingleScale}, values(object)) - end - return false -end - -_single_scale_runtime_object(object) = object -_single_scale_runtime_object(mapping::ModelMapping) = _modellist_from_model_mapping(mapping) - -function _modellist_from_model_mapping(mapping::ModelMapping{SingleScale}) +function _single_scale_model_set_from_mapping(mapping::ModelMapping{SingleScale}) mapping.data end -function _modellist_from_model_mapping(::ModelMapping{MultiScale}) +function _single_scale_model_set_from_mapping(::ModelMapping{MultiScale}) error("This `ModelMapping` is a multiscale mapping. ", "Use `run!(mtg, mapping, ...)` for multiscale mappings.") end -_modellist_from_model_mapping(mapping::ModelList) = mapping - function run!( - mapping::M, + mapping::ModelMapping{SingleScale}, meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; @@ -146,11 +132,11 @@ function run!( executor=ThreadedEx(), return_requested_outputs=false, requested_outputs_sink=DataFrames.DataFrame -) where {M<:Union{ModelMapping{SingleScale},ModelList}} +) _validate_meteo_duration(meteo) - model_list = _modellist_from_model_mapping(mapping) - _run_modellist_singleton( - model_list, + model_set = _single_scale_model_set_from_mapping(mapping) + _run_single_scale_model_set( + model_set, meteo, constants, extra; @@ -231,84 +217,11 @@ function run!( end ########################################################################################## -## ModelList (single-scale) simulations +## Single-scale simulations ########################################################################################## -# 1- several ModelList objects and several time-steps -function run!( - ::TableAlike, - object::T, - meteo::TimeStepTable{A}, - constants=PlantMeteo.Constants(), - extra=nothing; - tracked_outputs=nothing, - check=true, - executor=ThreadedEx(), - return_requested_outputs=false, - requested_outputs_sink=DataFrames.DataFrame -) where {T<:Union{AbstractArray,AbstractDict},A} - if _all_modellists_collection(object) - Base.depwarn( - "`run!` with a collection of `ModelList` is deprecated. Use a collection of `ModelMapping` objects instead.", - :run! - ) - end - - tracked_outputs isa OutputRequest && error("`OutputRequest` is only supported for MTG multi-rate simulations.") - tracked_outputs isa AbstractVector{<:OutputRequest} && error("`OutputRequest` is only supported for MTG multi-rate simulations.") - return_requested_outputs && error("`return_requested_outputs=true` is only supported for MTG multi-rate simulations.") - - if executor != SequentialEx() - @warn string( - "Parallelisation over objects was removed, (but may be reintroduced in the future). Parallelisation will only occur over timesteps." - ) maxlog = 1 - end - - outputs_collection = isa(object, AbstractArray) ? [] : isnothing(tracked_outputs) ? Dict() : Dict{TimeStepTable{Status{typeof(tracked_outputs)}}} - - # Each object: - for obj in object - - if isa(object, AbstractArray) - push!(outputs_collection, run!(obj, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor)) - else - outputs_collection[obj.first] = run!(obj.second, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor) - end - - end - return outputs_collection -end - # 2 - One object, one or multiple meteo time-step(s), with vectors provided in the status # (meaning a single meteo timestep might be expanded to fit the status vector size) -function run!( - ::SingletonAlike, - object::T, - meteo=nothing, - constants=PlantMeteo.Constants(), - extra=nothing; - tracked_outputs=nothing, - check=true, - executor=ThreadedEx(), - return_requested_outputs=false, - requested_outputs_sink=DataFrames.DataFrame -) where {T<:ModelList} - Base.depwarn( - "`run!(::ModelList, ...)` is deprecated. Use `run!(ModelMapping(...), ...)` instead.", - :run! - ) - _run_modellist_singleton( - object, - meteo, - constants, - extra; - tracked_outputs=tracked_outputs, - check=check, - executor=executor, - return_requested_outputs=return_requested_outputs - ) -end - function run!( ::SingletonAlike, object::T, @@ -321,10 +234,10 @@ function run!( return_requested_outputs=false, requested_outputs_sink=DataFrames.DataFrame ) where {T<:ModelMapping{SingleScale}} - model_list = _modellist_from_model_mapping(object) + model_set = _single_scale_model_set_from_mapping(object) - _run_modellist_singleton( - model_list, + _run_single_scale_model_set( + model_set, meteo, constants, extra; @@ -335,8 +248,8 @@ function run!( ) end -function _run_modellist_singleton( - object::ModelList, +function _run_single_scale_model_set( + object::SingleScaleModelSet, meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; @@ -352,6 +265,7 @@ function _run_modellist_singleton( meteo_adjusted = adjust_weather_timesteps_to_given_length(get_status_vector_max_length(object.status), meteo) nsteps = get_nsteps(meteo_adjusted) + validate_meteo_inputs(object.models, meteo_adjusted) dep_graph = dep!(object, nsteps) @@ -361,7 +275,7 @@ function _run_modellist_singleton( if length(dep_graph.not_found) > 0 error( - "The following processes are missing to run the ModelList: ", + "The following processes are missing to run the single-scale ModelMapping: ", dep_graph.not_found ) end @@ -429,72 +343,6 @@ function _run_modellist_singleton( return outputs_preallocated end -# 3- several objects and one meteo time-step -function run!( - ::TableAlike, - object::T, - meteo, - constants=PlantMeteo.Constants(), - extra=nothing; - tracked_outputs=nothing, - check=true, - executor=ThreadedEx(), - return_requested_outputs=false, - requested_outputs_sink=DataFrames.DataFrame -) where {T<:Union{AbstractArray,AbstractDict}} - if _all_modellists_collection(object) - Base.depwarn( - "`run!` with a collection of `ModelList` is deprecated. Use a collection of `ModelMapping` objects instead.", - :run! - ) - end - - tracked_outputs isa OutputRequest && error("`OutputRequest` is only supported for MTG multi-rate simulations.") - tracked_outputs isa AbstractVector{<:OutputRequest} && error("`OutputRequest` is only supported for MTG multi-rate simulations.") - return_requested_outputs && error("`return_requested_outputs=true` is only supported for MTG multi-rate simulations.") - runtime_objects = [_single_scale_runtime_object(obj) for obj in collect(values(object))] - dep_graphs = [dep(obj) for obj in runtime_objects] - #obj_parallelizable = all([object_parallelizable(graph) for graph in dep_graphs]) - - # Check if the simulation can be parallelized over objects: - if executor != SequentialEx() - @warn string( - "Parallelisation over objects was removed, (but may be reintroduced in the future). Parallelisation will only occur over timesteps." - ) maxlog = 1 - end - - # Each object: - for (i, obj) in enumerate(runtime_objects) - - if check - # Check if the meteo data and the status have the same length (or length 1) - check_dimensions(obj, meteo) - - if length(dep_graphs[i].not_found) > 0 - error( - "The following processes are missing to run the ModelList: ", - dep_graphs[i].not_found - ) - end - end - end - - outputs_collection = isa(object, AbstractArray) ? [] : isnothing(tracked_outputs) ? Dict() : Dict{TimeStepTable{Status{typeof(tracked_outputs)}}} - - # Each object: - for obj in object - if isa(object, AbstractArray) - push!(outputs_collection, run!(obj, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor)) - else - outputs_collection[obj.first] = run!(obj.second, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor) - end - - end - return outputs_collection -end - - - # Not exposed to the user : # for each dependency node in the graph (always one time-step, one object), actual workhorse function run_node!( @@ -505,7 +353,7 @@ function run_node!( meteo, constants, extra -) where {T<:ModelList} +) where {T<:SingleScaleModelSet} # Check if all the parents have been called before the child: if !AbstractTrees.isroot(node) && any([p.simulation_id[i] <= node.simulation_id[i] for p in node.parent]) @@ -605,40 +453,6 @@ function run!( return outputs(sim) end -function run!( - object::MultiScaleTreeGraph.Node, - mapping::AbstractDict{Symbol,T} where {T}, - meteo=nothing, - constants=PlantMeteo.Constants(), - extra=nothing; - nsteps=nothing, - tracked_outputs=nothing, - type_promotion=nothing, - check=true, - executor=ThreadedEx(), - return_requested_outputs=false, - requested_outputs_sink=DataFrames.DataFrame -) - Base.depwarn( - "`run!(mtg, mapping::AbstractDict, ...)` is deprecated. Use `run!(mtg, ModelMapping(mapping), ...)` or construct `ModelMapping(...)` directly.", - :run! - ) - run!( - object, - ModelMapping(mapping; type_promotion=type_promotion), - meteo, - constants, - extra; - nsteps=nsteps, - tracked_outputs=tracked_outputs, - type_promotion=type_promotion, - check=check, - executor=executor, - return_requested_outputs=return_requested_outputs, - requested_outputs_sink=requested_outputs_sink - ) -end - function run!( ::TreeAlike, object::GraphSimulation, @@ -661,6 +475,7 @@ function run!( runtime_clock_rows = _runtime_clock_rows(object, timeline, dep_graph) effective_executor = executor # st = status(object) + validate_meteo_inputs(get_model_specs(object), meteo) _validate_meteo_derived_timestep_requirements!(runtime_clock_rows, timeline) if effective_multirate if executor != SequentialEx() @@ -730,7 +545,10 @@ function run_node_multiscale!( executor, multirate, timeline::TimelineContext, - meteo_sampler + meteo_sampler; + meteo_provider=nothing, + after_model_run=nothing, + skip_model_run=nothing, ) where {T<:GraphSimulation} # T is the status of each node by organ type # run!(status(object), dependency_node, meteo, constants, extra) @@ -747,17 +565,27 @@ function run_node_multiscale!( model_clock = _model_clock(model_spec, node.value, timeline) t = _time_from_step(i, timeline) - for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) - should_run = !multirate || _should_run_at_time(model_clock, t) - !should_run && continue - if multirate - resolve_inputs_from_temporal_state!(object, node, st, t, model_spec, timeline) - end - meteo_for_model = multirate ? _sample_meteo_for_model(meteo_sampler, meteo, i, model_clock, model_spec) : meteo - # Actual call to the model: - run!(node.value, models_at_scale, st, meteo_for_model, constants, extra) - if multirate - update_temporal_state_outputs!(object, node, model_spec, st, t) + skip_node = !isnothing(skip_model_run) && skip_model_run(node) + if !skip_node + for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) + should_run = !multirate || _should_run_at_time(model_clock, t) + !should_run && continue + if multirate + resolve_inputs_from_temporal_state!(object, node, st, t, model_spec, timeline) + end + meteo_for_model = if isnothing(meteo_provider) + multirate ? _sample_meteo_for_model(meteo_sampler, meteo, i, model_clock, model_spec) : meteo + else + meteo_provider(node, st, i, t, model_clock, model_spec, meteo, meteo_sampler, multirate) + end + # Actual call to the model: + run!(node.value, models_at_scale, st, meteo_for_model, constants, extra) + if !isnothing(after_model_run) + after_model_run(node, model_spec, st, i, t) + end + if multirate + update_temporal_state_outputs!(object, node, model_spec, st, t) + end end end @@ -768,6 +596,22 @@ function run_node_multiscale!( #! check if we can run this safely in a @floop loop. I would say no, #! because we are running a parallel computation above already, modifying the node.simulation_id, #! which is not thread-safe yet. - run_node_multiscale!(object, child, i, models, meteo, constants, extra, check, executor, multirate, timeline, meteo_sampler) + run_node_multiscale!( + object, + child, + i, + models, + meteo, + constants, + extra, + check, + executor, + multirate, + timeline, + meteo_sampler; + meteo_provider=meteo_provider, + after_model_run=after_model_run, + skip_model_run=skip_model_run, + ) end end diff --git a/src/scene_object_api.jl b/src/scene_object_api.jl new file mode 100644 index 000000000..9959b7903 --- /dev/null +++ b/src/scene_object_api.jl @@ -0,0 +1,2772 @@ +abstract type AbstractObjectSelector end +abstract type AbstractObjectMultiplicity end + +struct ObjectRefVector{R} <: AbstractVector{Any} + refs::R +end + +Base.size(v::ObjectRefVector) = size(v.refs) +Base.length(v::ObjectRefVector) = length(v.refs) +Base.getindex(v::ObjectRefVector, i::Int) = v.refs[i][] +Base.setindex!(v::ObjectRefVector, value, i::Int) = (v.refs[i][] = value) +Base.parent(v::ObjectRefVector) = v.refs + +struct ObjectId{T} + value::T +end +ObjectId(id::ObjectId) = id +ObjectId(id::AbstractString) = ObjectId(Symbol(id)) + +mutable struct Object + id::ObjectId + scale::Union{Nothing,Symbol} + kind::Union{Nothing,Symbol} + species::Union{Nothing,Symbol} + name::Union{Nothing,Symbol} + parent::Union{Nothing,ObjectId} + children::Vector{ObjectId} + geometry::Any + status::Any + applications::Any +end + +""" + ObjectTemplate(applications=(); kind=nothing, species=nothing, mapping=applications, parameters=NamedTuple()) + +Reusable model-application bundle for one kind of scene object, such as a plant +species. Each mounted `ObjectInstance` scopes unqualified `AppliesTo(...)` +selectors to its own object subtree. Model objects are shared between instances +unless an instance supplies an override. + +`parameters` stores template-level metadata. Parameter-field merging is not +implicit: use an instance model override when model parameters differ. +""" +struct ObjectTemplate{A,P} + kind::Union{Nothing,Symbol} + species::Union{Nothing,Symbol} + applications::A + parameters::P +end + +_as_tuple(value::Tuple) = value +_as_tuple(value::AbstractVector) = Tuple(value) +_as_tuple(value) = (value,) + +function ObjectTemplate( + applications=(); + kind=nothing, + species=nothing, + mapping=applications, + parameters=NamedTuple(), +) + normalized_applications = _as_tuple(mapping) + return ObjectTemplate( + _maybe_symbol(kind), + _maybe_symbol(species), + normalized_applications, + parameters, + ) +end + +""" + Override(; object, model, process=nothing, application=nothing) + +Replace one template model application on one exceptional object. Select the +template application by its process, explicit application name, or both. +The replacement must implement the same process and variable contract. +""" +struct Override{M<:AbstractModel} + object::ObjectId + process::Union{Nothing,Symbol} + application::Union{Nothing,Symbol} + model::M +end + +function Override(; object, model::AbstractModel, process=nothing, application=nothing) + normalized_process = _maybe_symbol(process) + normalized_application = _maybe_symbol(application) + if isnothing(normalized_process) && isnothing(normalized_application) + error("`Override(...)` requires `process=...`, `application=...`, or both.") + end + return Override( + ObjectId(object), + normalized_process, + normalized_application, + model, + ) +end + +""" + ObjectInstance(name, template; root, objects=(), overrides=NamedTuple(), object_overrides=()) + +Mount an `ObjectTemplate` on one concrete scene-object subtree. + +`root` may be an `Object` owned by the instance or the id of an object supplied +separately to `Scene`. `objects` contains additional owned descendants. +`overrides` maps one template application name or process to a replacement +model implementing the same process. `object_overrides` contains `Override` +entries for exceptional organs. +""" +struct ObjectInstance{T,R,O,OV,OOV} + name::Symbol + template::T + root::R + objects::O + overrides::OV + object_overrides::OOV +end + +function ObjectInstance( + name, + template::ObjectTemplate; + root, + objects=(), + overrides=NamedTuple(), + object_overrides=(), +) + normalized_objects = _as_tuple(objects) + normalized_object_overrides = _as_tuple(object_overrides) + all(object -> object isa Object, normalized_objects) || error( + "`ObjectInstance(...; objects=...)` must contain `Object` values." + ) + overrides isa NamedTuple || error( + "`ObjectInstance(...; overrides=...)` must be a NamedTuple keyed by application name or process." + ) + all(override -> override isa Override, normalized_object_overrides) || error( + "`ObjectInstance(...; object_overrides=...)` must contain `Override` values." + ) + return ObjectInstance( + Symbol(name), + template, + root, + normalized_objects, + overrides, + normalized_object_overrides, + ) +end + +struct ObjectModelOverrides{M,O} <: AbstractModel + base::M + overrides::O +end + +process(model::ObjectModelOverrides) = process(model.base) +inputs_(model::ObjectModelOverrides) = inputs_(model.base) +outputs_(model::ObjectModelOverrides) = outputs_(model.base) +dep(model::ObjectModelOverrides, nsteps=1) = dep(model.base, nsteps) +timespec(model::ObjectModelOverrides) = timespec(model.base) +output_policy(model::ObjectModelOverrides) = output_policy(model.base) +meteo_inputs_(model::ObjectModelOverrides) = meteo_inputs_(model.base) +meteo_outputs_(model::ObjectModelOverrides) = meteo_outputs_(model.base) + +function Object( + id; + scale=nothing, + kind=nothing, + species=nothing, + name=nothing, + parent=nothing, + children=ObjectId[], + geometry=nothing, + status=nothing, + applications=(), +) + return Object( + ObjectId(id), + _maybe_symbol(scale), + _maybe_symbol(kind), + _maybe_symbol(species), + _maybe_symbol(name), + isnothing(parent) ? nothing : ObjectId(parent), + ObjectId[ObjectId(child) for child in children], + geometry, + status, + applications, + ) +end + +mutable struct SceneRegistry + objects::Dict{ObjectId,Any} + by_scale::Dict{Symbol,Set{ObjectId}} + by_kind::Dict{Symbol,Set{ObjectId}} + by_species::Dict{Symbol,Set{ObjectId}} + by_name::Dict{Symbol,ObjectId} +end + +SceneRegistry() = SceneRegistry( + Dict{ObjectId,Any}(), + Dict{Symbol,Set{ObjectId}}(), + Dict{Symbol,Set{ObjectId}}(), + Dict{Symbol,Set{ObjectId}}(), + Dict{Symbol,ObjectId}(), +) + +mutable struct Scene{R,A,E,I} + registry::R + applications::A + environment::E + instances::I + binding_cache::Any + environment_binding_cache::Any + bindings_dirty::Bool + environment_bindings_dirty::Bool + revision::Int + environment_revision::Int +end + +function _normalize_object_instances(instances) + instances isa ObjectInstance && return (instances,) + normalized = _as_tuple(instances) + all(instance -> instance isa ObjectInstance, normalized) || error( + "Scene instances must be `ObjectInstance` values." + ) + return normalized +end + +function _instance_root_id(instance::ObjectInstance) + return instance.root isa Object ? instance.root.id : ObjectId(instance.root) +end + +function _collect_scene_items(items, instances) + objects = Object[] + mounted_instances = ObjectInstance[] + for item in items + if item isa Object + push!(objects, item) + elseif item isa ObjectInstance + push!(mounted_instances, item) + else + error("A `Scene` can contain only `Object` and `ObjectInstance` values, got `$(typeof(item))`.") + end + end + append!(mounted_instances, _normalize_object_instances(instances)) + for instance in mounted_instances + instance.root isa Object && push!(objects, instance.root) + append!(objects, instance.objects) + end + ids = Set{ObjectId}() + for object in objects + object.id in ids && error("Scene contains object id `$(object.id.value)` more than once.") + push!(ids, object.id) + end + return objects, mounted_instances +end + +function _object_descendant_ids(objects_by_id, root_id::ObjectId) + ids = ObjectId[root_id] + frontier = ObjectId[root_id] + while !isempty(frontier) + parent_id = popfirst!(frontier) + for object in values(objects_by_id) + object.parent == parent_id || continue + object.id in ids && continue + push!(ids, object.id) + push!(frontier, object.id) + end + end + return ids +end + +function _prepare_object_instances!(objects, instances) + objects_by_id = Dict(object.id => object for object in objects) + claimed_ids = Dict{ObjectId,Symbol}() + instance_ids = Dict{Symbol,Vector{ObjectId}}() + for instance in instances + root_id = _instance_root_id(instance) + haskey(objects_by_id, root_id) || error( + "Object instance `$(instance.name)` refers to missing root object `$(root_id.value)`." + ) + ids = _object_descendant_ids(objects_by_id, root_id) + for id in ids + if haskey(claimed_ids, id) + error( + "Object `$(id.value)` belongs to both instances `$(claimed_ids[id])` and `$(instance.name)`." + ) + end + claimed_ids[id] = instance.name + object = objects_by_id[id] + isnothing(object.kind) && (object.kind = instance.template.kind) + isnothing(object.species) && (object.species = instance.template.species) + end + root = objects_by_id[root_id] + if !isnothing(root.name) && root.name != instance.name + error( + "Object instance `$(instance.name)` root `$(root_id.value)` already has the conflicting name `$(root.name)`." + ) + end + root.name = instance.name + instance_ids[instance.name] = ids + end + length(instance_ids) == length(instances) || error("Object instance names must be unique within a scene.") + return instance_ids +end + +function _register_scene_objects!(scene::Scene, objects) + pending = copy(objects) + while !isempty(pending) + registered = false + for index in reverse(eachindex(pending)) + object = pending[index] + if isnothing(object.parent) || haskey(scene.registry.objects, object.parent) + register_object!(scene, object) + deleteat!(pending, index) + registered = true + end + end + registered && continue + unresolved = [(object.id.value, isnothing(object.parent) ? nothing : object.parent.value) for object in pending] + error("Cannot register scene objects because parent objects are missing or cyclic: $(unresolved).") + end + return scene +end + +""" + Scene(items...; applications=(), instances=(), environment=nothing) + +Create a scene from `Object` and `ObjectInstance` values. Global applications +and applications mounted from object instances are compiled through the same +scene/object dependency graph. +""" +function Scene( + items::Union{Object,ObjectInstance}...; + applications=(), + instances=(), + environment=nothing, +) + objects, mounted_instances = _collect_scene_items(items, instances) + instance_ids = _prepare_object_instances!(objects, mounted_instances) + mounted_applications = _mount_object_instance_applications(mounted_instances, instance_ids) + normalized_applications = _as_tuple(applications) + all_applications = (normalized_applications..., mounted_applications...) + scene = Scene( + SceneRegistry(), + all_applications, + environment, + mounted_instances, + nothing, + nothing, + true, + true, + 0, + 0, + ) + return _register_scene_objects!(scene, objects) +end + +function _mark_environment_bindings_dirty!(scene::Scene) + scene.environment_binding_cache = nothing + scene.environment_bindings_dirty = true + scene.environment_revision += 1 + return scene +end + +function _mark_bindings_dirty!(scene::Scene) + scene.binding_cache = nothing + scene.bindings_dirty = true + scene.revision += 1 + return _mark_environment_bindings_dirty!(scene) +end + +bindings_dirty(scene::Scene) = scene.bindings_dirty +environment_bindings_dirty(scene::Scene) = scene.environment_bindings_dirty +scene_revision(scene::Scene) = scene.revision +environment_revision(scene::Scene) = scene.environment_revision +compiled_bindings(scene::Scene) = scene.binding_cache +compiled_environment_bindings(scene::Scene) = scene.environment_binding_cache +mark_environment_binding_dirty!(scene::Scene) = _mark_environment_bindings_dirty!(scene) +function mark_environment_binding_dirty!(scene::Scene, id) + _scene_object(scene, id) + return _mark_environment_bindings_dirty!(scene) +end +function mark_environment_binding_dirty!(scene::Scene, object::Object) + return mark_environment_binding_dirty!(scene, object.id) +end + +function _push_index!(index::Dict{Symbol,Set{ObjectId}}, key, id::ObjectId) + isnothing(key) && return nothing + push!(get!(index, key, Set{ObjectId}()), id) + return nothing +end + +function _delete_index!(index::Dict{Symbol,Set{ObjectId}}, key, id::ObjectId) + isnothing(key) && return nothing + ids = get(index, key, nothing) + isnothing(ids) && return nothing + delete!(ids, id) + isempty(ids) && delete!(index, key) + return nothing +end + +function _index_object!(registry::SceneRegistry, object::Object) + _push_index!(registry.by_scale, object.scale, object.id) + _push_index!(registry.by_kind, object.kind, object.id) + _push_index!(registry.by_species, object.species, object.id) + if !isnothing(object.name) + existing = get(registry.by_name, object.name, nothing) + if !isnothing(existing) && existing != object.id + error( + "Scene object name `$(object.name)` is already used by object `$(existing.value)`." + ) + end + registry.by_name[object.name] = object.id + end + return nothing +end + +function _deindex_object!(registry::SceneRegistry, object::Object) + _delete_index!(registry.by_scale, object.scale, object.id) + _delete_index!(registry.by_kind, object.kind, object.id) + _delete_index!(registry.by_species, object.species, object.id) + if !isnothing(object.name) && get(registry.by_name, object.name, nothing) == object.id + delete!(registry.by_name, object.name) + end + return nothing +end + +function _scene_object(scene::Scene, id) + oid = ObjectId(id) + haskey(scene.registry.objects, oid) || error("No scene object with id `$(oid.value)`.") + return scene.registry.objects[oid] +end + +function _instance_for_object(scene::Scene, id) + current_id = ObjectId(id) + while haskey(scene.registry.objects, current_id) + for instance in scene.instances + _instance_root_id(instance) == current_id && return instance + end + parent = scene.registry.objects[current_id].parent + isnothing(parent) && return nothing + current_id = parent + end + return nothing +end + +function _apply_instance_labels!(object::Object, instance) + isnothing(instance) && return object + isnothing(object.kind) && (object.kind = instance.template.kind) + isnothing(object.species) && (object.species = instance.template.species) + return object +end + +function register_object!(scene::Scene, object::Object; parent=object.parent) + registry = scene.registry + haskey(registry.objects, object.id) && error("Scene already contains object id `$(object.id.value)`.") + parent_id = isnothing(parent) ? nothing : ObjectId(parent) + if !isnothing(parent_id) && !haskey(registry.objects, parent_id) + error("No scene object with id `$(parent_id.value)`.") + end + if !isnothing(object.name) + existing = get(registry.by_name, object.name, nothing) + isnothing(existing) || error( + "Scene object name `$(object.name)` is already used by object `$(existing.value)`." + ) + end + instance = isnothing(parent_id) ? nothing : _instance_for_object(scene, parent_id) + _apply_instance_labels!(object, instance) + object.parent = parent_id + registry.objects[object.id] = object + _index_object!(registry, object) + if !isnothing(object.parent) + parent_object = registry.objects[object.parent] + object.id in parent_object.children || push!(parent_object.children, object.id) + end + _mark_bindings_dirty!(scene) + return object +end + +function _remove_child_link!(scene::Scene, parent_id, child_id::ObjectId) + isnothing(parent_id) && return nothing + parent_object = _scene_object(scene, parent_id) + filter!(!=(child_id), parent_object.children) + return nothing +end + +function remove_object!(scene::Scene, id; recursive::Bool=true) + object = _scene_object(scene, id) + if !recursive && !isempty(object.children) + error("Cannot remove object `$(object.id.value)` with children unless `recursive=true`.") + end + for child in copy(object.children) + remove_object!(scene, child; recursive=true) + end + _remove_child_link!(scene, object.parent, object.id) + _deindex_object!(scene.registry, object) + delete!(scene.registry.objects, object.id) + _mark_bindings_dirty!(scene) + return object +end + +function reparent_object!(scene::Scene, id, new_parent) + object = _scene_object(scene, id) + new_parent_id = isnothing(new_parent) ? nothing : ObjectId(new_parent) + if !isnothing(new_parent_id) + haskey(scene.registry.objects, new_parent_id) || error("No scene object with id `$(new_parent_id.value)`.") + end + _remove_child_link!(scene, object.parent, object.id) + object.parent = new_parent_id + if !isnothing(new_parent_id) + parent_object = _scene_object(scene, new_parent_id) + object.id in parent_object.children || push!(parent_object.children, object.id) + end + _mark_bindings_dirty!(scene) + return object +end + +function move_object!(scene::Scene, id, geometry_or_position) + return update_geometry!(scene, id, geometry_or_position) +end + +function update_geometry!(scene::Scene, id, geometry_or_position; invalidate_environment::Bool=true) + object = _scene_object(scene, id) + object.geometry = geometry_or_position + invalidate_environment && _mark_environment_bindings_dirty!(scene) + return object +end + +function update_geometry!(object::Object, geometry_or_position) + object.geometry = geometry_or_position + return object +end + +geometry(object::Object) = object.geometry +geometry(status::Status) = hasproperty(status, :geometry) ? status.geometry : nothing +geometry(x) = hasproperty(x, :geometry) ? getproperty(x, :geometry) : nothing + +function _geometry_position(g::NamedTuple) + haskey(g, :position) && return getproperty(g, :position) + if haskey(g, :x) && haskey(g, :y) && haskey(g, :z) + return (x=g.x, y=g.y, z=g.z) + elseif haskey(g, :x) && haskey(g, :y) + return (x=g.x, y=g.y) + end + return nothing +end +_geometry_position(_) = nothing + +position(object::Object) = _geometry_position(geometry(object)) +position(status::Status) = _geometry_position(geometry(status)) + +_geometry_bounds(g::NamedTuple) = haskey(g, :bounds) ? getproperty(g, :bounds) : nothing +_geometry_bounds(_) = nothing +bounds(object::Object) = _geometry_bounds(geometry(object)) +bounds(status::Status) = _geometry_bounds(geometry(status)) + +function refresh_bindings!(scene::Scene, specs=scene.applications; force::Bool=false) + uses_scene_applications = specs === scene.applications + if !uses_scene_applications + return compile_scene(scene, specs) + end + if force || scene.bindings_dirty || isnothing(scene.binding_cache) + scene.binding_cache = compile_scene(scene, scene.applications) + scene.bindings_dirty = false + end + return scene.binding_cache +end + +function refresh_environment_bindings!(scene::Scene, compiled=refresh_bindings!(scene); force::Bool=false) + if force || scene.environment_bindings_dirty || isnothing(scene.environment_binding_cache) + scene.environment_binding_cache = compile_environment_bindings(scene, compiled) + scene.environment_bindings_dirty = false + end + return scene.environment_binding_cache +end + +function object_ids(scene::Scene; scale=nothing, kind=nothing, species=nothing, name=nothing) + registry = scene.registry + sets = Set{ObjectId}[] + isnothing(scale) || push!(sets, copy(get(registry.by_scale, Symbol(scale), Set{ObjectId}()))) + isnothing(kind) || push!(sets, copy(get(registry.by_kind, Symbol(kind), Set{ObjectId}()))) + isnothing(species) || push!(sets, copy(get(registry.by_species, Symbol(species), Set{ObjectId}()))) + if !isnothing(name) + id = get(registry.by_name, Symbol(name), nothing) + push!(sets, isnothing(id) ? Set{ObjectId}() : Set([id])) + end + isempty(sets) && return sort!(collect(keys(registry.objects)); by=id -> string(id.value)) + ids = reduce(intersect, sets) + return sort!(collect(ids); by=id -> string(id.value)) +end + +scene_objects(scene::Scene; kwargs...) = [_scene_object(scene, id) for id in object_ids(scene; kwargs...)] + +function _instance_object_ids(scene::Scene, instance::ObjectInstance) + root_id = _instance_root_id(instance) + haskey(scene.registry.objects, root_id) || return ObjectId[] + return _sort_object_ids!(_descendant_ids(scene, root_id)) +end + +function _object_instance_name(scene::Scene, object_id::ObjectId) + instance = _instance_for_object(scene, object_id) + return isnothing(instance) ? nothing : instance.name +end + +function explain_objects(scene::Scene) + return [ + ( + id=object.id.value, + scale=object.scale, + kind=object.kind, + species=object.species, + name=object.name, + instance=_object_instance_name(scene, object.id), + parent=isnothing(object.parent) ? nothing : object.parent.value, + children=[child.value for child in object.children], + has_geometry=!isnothing(object.geometry), + has_status=!isnothing(object.status), + n_applications=length(object.applications), + ) + for object in scene_objects(scene) + ] +end + +function _instance_application_ids(scene::Scene, instance::ObjectInstance) + prefix = string(instance.name, "__") + ids = Symbol[] + for application in scene.applications + name = application_name(as_model_spec(application)) + isnothing(name) && continue + startswith(string(name), prefix) && push!(ids, name) + end + return sort!(ids; by=string) +end + +function explain_instances(scene::Scene) + return [ + ( + name=instance.name, + root_id=_instance_root_id(instance).value, + kind=instance.template.kind, + species=instance.template.species, + object_ids=[id.value for id in _instance_object_ids(scene, instance)], + application_ids=_instance_application_ids(scene, instance), + instance_overrides=sort!(Symbol[Symbol(name) for name in keys(instance.overrides)]; by=string), + object_overrides=[ + ( + object_id=override.object.value, + process=override.process, + application=override.application, + model_type=typeof(override.model), + ) + for override in instance.object_overrides + ], + parameters_type=typeof(instance.template.parameters), + parameters_shared_by_reference=true, + ) + for instance in scene.instances + ] +end + +function _object_id_values(ids) + return [id.value for id in _sort_object_ids!(collect(ids))] +end + +function _label_scope_rows(scene::Scene, scope_type::Symbol, label::Symbol, index) + return [ + ( + scope_type=scope_type, + selector=label => key, + context=nothing, + root_id=nothing, + scale=label == :scale ? key : nothing, + kind=label == :kind ? key : nothing, + species=label == :species ? key : nothing, + name=nothing, + object_ids=_object_id_values(ids), + n_objects=length(ids), + ) + for (key, ids) in sort!(collect(index); by=pair -> string(first(pair))) + ] +end + +function explain_scopes(scene::Scene) + rows = NamedTuple[] + all_ids = object_ids(scene) + push!( + rows, + ( + scope_type=:scene, + selector=SceneScope(), + context=nothing, + root_id=nothing, + scale=nothing, + kind=nothing, + species=nothing, + name=nothing, + object_ids=[id.value for id in all_ids], + n_objects=length(all_ids), + ), + ) + for object in scene_objects(scene) + descendant_ids = _object_id_values(_descendant_ids(scene, object.id)) + push!( + rows, + ( + scope_type=:object_subtree, + selector=Self(), + context=object.id.value, + root_id=object.id.value, + scale=object.scale, + kind=object.kind, + species=object.species, + name=object.name, + object_ids=descendant_ids, + n_objects=length(descendant_ids), + ), + ) + object_scope_name = object.id.value isa Symbol ? object.id.value : Symbol(string(object.id.value)) + scope_names = Symbol[object_scope_name] + isnothing(object.name) || push!(scope_names, object.name) + unique!(scope_names) + for scope_name in scope_names + push!( + rows, + ( + scope_type=:named_scope, + selector=Scope(scope_name), + context=nothing, + root_id=object.id.value, + scale=object.scale, + kind=object.kind, + species=object.species, + name=scope_name, + object_ids=descendant_ids, + n_objects=length(descendant_ids), + ), + ) + end + end + append!(rows, _label_scope_rows(scene, :scale, :scale, scene.registry.by_scale)) + append!(rows, _label_scope_rows(scene, :kind, :kind, scene.registry.by_kind)) + append!(rows, _label_scope_rows(scene, :species, :species, scene.registry.by_species)) + return rows +end + +struct SceneScope <: AbstractObjectSelector end +struct Self <: AbstractObjectSelector end +struct SelfPlant <: AbstractObjectSelector end + +struct Ancestor <: AbstractObjectSelector + scale::Union{Nothing,Symbol} +end +Ancestor(; scale=nothing) = Ancestor(_maybe_symbol(scale)) + +struct Scope <: AbstractObjectSelector + name::Symbol +end +Scope(name::Union{Symbol,AbstractString}) = Scope(Symbol(name)) + +struct Kind <: AbstractObjectSelector + kind::Symbol +end +Kind(kind::Union{Symbol,AbstractString}) = Kind(Symbol(kind)) + +struct Species <: AbstractObjectSelector + species::Symbol +end +Species(species::Union{Symbol,AbstractString}) = Species(Symbol(species)) + +struct Scale <: AbstractObjectSelector + scale::Symbol +end +Scale(scale::Union{Symbol,AbstractString}) = Scale(Symbol(scale)) + +struct Relation <: AbstractObjectSelector + relation::Symbol +end +Relation(relation::Union{Symbol,AbstractString}) = Relation(Symbol(relation)) + +_maybe_symbol(x) = isnothing(x) ? nothing : Symbol(x) + +const _OBJECT_ADDRESS_SYMBOL_FIELDS = (:kind, :domain, :species, :scale, :name, :process, :var, :relation, :application) + +function _normalize_object_selector_value(key::Symbol, value) + key in _OBJECT_ADDRESS_SYMBOL_FIELDS && return _maybe_symbol(value) + return value +end + +function _normalize_selector_kwargs(kwargs) + return NamedTuple{keys(kwargs)}( + Tuple(_normalize_object_selector_value(k, v) for (k, v) in pairs(kwargs)) + ) +end + +function _normalize_selector_criteria(args::Tuple; kwargs...) + selectors = Tuple(args) + all(selector -> selector isa AbstractObjectSelector, selectors) || error( + "Object selector positional arguments must be selector objects such as `Kind(:plant)` or `Scale(:Leaf)`." + ) + normalized_kwargs = _normalize_selector_kwargs((; kwargs...)) + return (; selectors=selectors, normalized_kwargs...) +end + +struct One{C<:NamedTuple} <: AbstractObjectMultiplicity + criteria::C +end +struct OptionalOne{C<:NamedTuple} <: AbstractObjectMultiplicity + criteria::C +end +struct Many{C<:NamedTuple} <: AbstractObjectMultiplicity + criteria::C +end + +One(args...; kwargs...) = One(_normalize_selector_criteria(args; kwargs...)) +OptionalOne(args...; kwargs...) = OptionalOne(_normalize_selector_criteria(args; kwargs...)) +Many(args...; kwargs...) = Many(_normalize_selector_criteria(args; kwargs...)) + +criteria(selector::AbstractObjectMultiplicity) = selector.criteria +multiplicity(::One) = :one +multiplicity(::OptionalOne) = :optional_one +multiplicity(::Many) = :many + +_rebuild_selector(::One, criteria) = One{typeof(criteria)}(criteria) +_rebuild_selector(::OptionalOne, criteria) = OptionalOne{typeof(criteria)}(criteria) +_rebuild_selector(::Many, criteria) = Many{typeof(criteria)}(criteria) + +function _selector_with_scope(selector::AbstractObjectMultiplicity, scope) + selector_criteria = criteria(selector) + haskey(selector_criteria, :within) && return selector + return _rebuild_selector(selector, merge(selector_criteria, (; within=scope))) +end + +function _selector_with_application_prefix( + selector::AbstractObjectMultiplicity, + instance_name::Symbol, + template_application_names::Set{Symbol}, +) + selector_criteria = criteria(selector) + haskey(selector_criteria, :application) || return selector + application = selector_criteria.application + isnothing(application) && return selector + application in template_application_names || return selector + mounted_name = Symbol(instance_name, "__", application) + return _rebuild_selector(selector, merge(selector_criteria, (; application=mounted_name))) +end + +function _mounted_application_name(spec, index::Int) + name = application_name(spec) + return isnothing(name) ? process(spec) : name +end + +function _instance_override_matches(spec, key::Symbol) + name = application_name(spec) + return key == process(spec) || (!isnothing(name) && key == name) +end + +function _instance_override_models(instance::ObjectInstance, specs) + selected = Dict{Int,AbstractModel}() + for (key, replacement) in pairs(instance.overrides) + replacement isa AbstractModel || error( + "Override `$(key)` for instance `$(instance.name)` must be an `AbstractModel`, got `$(typeof(replacement))`." + ) + matches = findall(spec -> _instance_override_matches(spec, Symbol(key)), specs) + isempty(matches) && error( + "Override `$(key)` for instance `$(instance.name)` does not match a template application name or process." + ) + length(matches) == 1 || error( + "Override `$(key)` for instance `$(instance.name)` matches several template applications; use a unique application name." + ) + index = only(matches) + haskey(selected, index) && error( + "Several overrides target template application `$(_mounted_application_name(specs[index], index))` in instance `$(instance.name)`." + ) + _validate_model_override_contract!( + model_(specs[index]), + replacement; + description="Override `$(key)` for instance `$(instance.name)`", + ) + selected[index] = replacement + end + return selected +end + +function _model_contract(model) + return ( + process=process(model), + inputs=Tuple(Symbol.(keys(inputs_(model)))), + outputs=Tuple(Symbol.(keys(outputs_(model)))), + meteo_inputs=Tuple(Symbol.(keys(meteo_inputs_(model)))), + meteo_outputs=Tuple(Symbol.(keys(meteo_outputs_(model)))), + ) +end + +function _validate_model_override_contract!(base, replacement; description) + base_contract = _model_contract(base) + replacement_contract = _model_contract(replacement) + base_contract == replacement_contract && return nothing + error( + "$(description) has an incompatible model contract. Expected ", + "`$(base_contract)`, got `$(replacement_contract)`. Object and instance ", + "overrides may change parameters or implementation, but not process or declared variables." + ) +end + +function _object_override_matches(spec, override::Override) + process_match = isnothing(override.process) || process(spec) == override.process + name = application_name(spec) + application_match = isnothing(override.application) || + (!isnothing(name) && name == override.application) + return process_match && application_match +end + +function _object_override_models(instance::ObjectInstance, specs, instance_ids) + entries = Dict{Int,Vector{Pair{ObjectId,AbstractModel}}}() + valid_ids = Set(instance_ids) + for override in instance.object_overrides + override.object in valid_ids || error( + "Object override for `$(override.object.value)` does not belong to instance `$(instance.name)`." + ) + matches = findall(spec -> _object_override_matches(spec, override), specs) + isempty(matches) && error( + "Object override for `$(override.object.value)` in instance `$(instance.name)` ", + "does not match a template application." + ) + length(matches) == 1 || error( + "Object override for `$(override.object.value)` in instance `$(instance.name)` ", + "matches several template applications; add `application=...`." + ) + index = only(matches) + object_models = get!(entries, index, Pair{ObjectId,AbstractModel}[]) + any(entry -> first(entry) == override.object, object_models) && error( + "Several object overrides target application `$(_mounted_application_name(specs[index], index))` ", + "on object `$(override.object.value)` in instance `$(instance.name)`." + ) + _validate_model_override_contract!( + model_(specs[index]), + override.model; + description="Object override for `$(override.object.value)` in instance `$(instance.name)`", + ) + push!(object_models, override.object => override.model) + end + selected = Dict{Int,Any}() + for (index, object_models) in entries + selected[index] = _typed_object_model_dict(object_models) + end + return selected +end + +function _typed_object_model_dict(entries) + isempty(entries) && return Dict{ObjectId,AbstractModel}() + model_type = typeof(last(first(entries))) + if all(entry -> typeof(last(entry)) == model_type, entries) + models = Dict{ObjectId,model_type}() + for (object_id, model) in entries + models[object_id] = model + end + return models + end + return Dict{ObjectId,AbstractModel}(entries) +end + +function _map_selector_bindings(bindings::NamedTuple, f) + mapped = Pair{Symbol,Any}[] + for (name, selector) in pairs(bindings) + push!(mapped, Symbol(name) => (selector isa AbstractObjectMultiplicity ? f(selector) : selector)) + end + return (; mapped...) +end + +function _mount_object_instance_applications(instance::ObjectInstance, instance_ids) + specs = Tuple(as_model_spec(application) for application in instance.template.applications) + base_names = Set(_mounted_application_name(spec, index) for (index, spec) in pairs(specs)) + instance_overrides = _instance_override_models(instance, specs) + object_overrides = _object_override_models(instance, specs, instance_ids) + mounted = Any[] + for (index, spec) in pairs(specs) + base_name = _mounted_application_name(spec, index) + mounted_name = Symbol(instance.name, "__", base_name) + target = applies_to(spec) + isnothing(target) && error( + "Template application `$(base_name)` has no `AppliesTo(...)` selector." + ) + mounted_target = _selector_with_scope(target, Scope(instance.name)) + prefix_application = selector -> _selector_with_application_prefix( + selector, + instance.name, + base_names, + ) + mounted_inputs = _map_selector_bindings(value_inputs(spec), prefix_application) + mounted_calls = _map_selector_bindings(model_calls(spec), prefix_application) + mounted_model = get(instance_overrides, index, model_(spec)) + if haskey(object_overrides, index) + object_models = object_overrides[index] + for replacement in values(object_models) + _validate_model_override_contract!( + mounted_model, + replacement; + description="Object override for template application `$(base_name)`", + ) + end + mounted_model = ObjectModelOverrides(mounted_model, object_models) + end + push!( + mounted, + ModelSpec( + spec; + model=mounted_model, + name=mounted_name, + applies_to=mounted_target, + inputs=mounted_inputs, + calls=mounted_calls, + ), + ) + end + return Tuple(mounted) +end + +function _mount_object_instance_applications(instances, instance_ids) + mounted = Any[] + for instance in instances + append!( + mounted, + _mount_object_instance_applications(instance, instance_ids[instance.name]), + ) + end + return Tuple(mounted) +end + +_sort_object_ids!(ids) = sort!(ids; by=id -> string(id.value)) + +function _object_id_from_context(context) + isnothing(context) && return nothing + context isa Object && return context.id + return ObjectId(context) +end + +function _descendant_ids(scene::Scene, root_id::ObjectId) + ids = ObjectId[root_id] + object = _scene_object(scene, root_id) + for child_id in object.children + append!(ids, _descendant_ids(scene, child_id)) + end + return ids +end + +function _ancestor_id(scene::Scene, current_id::ObjectId; scale=nothing, kind=nothing) + id = current_id + while true + object = _scene_object(scene, id) + scale_match = isnothing(scale) || object.scale == Symbol(scale) + kind_match = isnothing(kind) || object.kind == Symbol(kind) + scale_match && kind_match && return id + isnothing(object.parent) && return nothing + id = object.parent + end +end + +function _selector_scope_from_positional(selectors) + scopes = filter( + selector -> selector isa Union{SceneScope,Self,SelfPlant,Ancestor,Scope}, + selectors, + ) + isempty(scopes) && return nothing + length(scopes) == 1 || error("Only one scope selector can be used in one object selector.") + return only(scopes) +end + +function _selector_value_from_positional(selectors, ::Type{Kind}) + values = [selector.kind for selector in selectors if selector isa Kind] + isempty(values) && return nothing + length(unique(values)) == 1 || error("Conflicting `Kind(...)` selector values: $(values).") + return only(unique(values)) +end + +function _selector_value_from_positional(selectors, ::Type{Species}) + values = [selector.species for selector in selectors if selector isa Species] + isempty(values) && return nothing + length(unique(values)) == 1 || error("Conflicting `Species(...)` selector values: $(values).") + return only(unique(values)) +end + +function _selector_value_from_positional(selectors, ::Type{Scale}) + values = [selector.scale for selector in selectors if selector isa Scale] + isempty(values) && return nothing + length(unique(values)) == 1 || error("Conflicting `Scale(...)` selector values: $(values).") + return only(unique(values)) +end + +function _selector_value_from_positional(selectors, ::Type{Relation}) + values = [selector.relation for selector in selectors if selector isa Relation] + isempty(values) && return nothing + length(unique(values)) == 1 || error("Conflicting `Relation(...)` selector values: $(values).") + return only(unique(values)) +end + +function _criteria_value(criteria, key::Symbol, selector_type) + positional = _selector_value_from_positional(criteria.selectors, selector_type) + keyword = haskey(criteria, key) ? getproperty(criteria, key) : nothing + if !isnothing(positional) && !isnothing(keyword) && positional != keyword + error("Conflicting selector values for `$(key)`: `$(positional)` and `$(keyword)`.") + end + return isnothing(keyword) ? positional : keyword +end + +function _criteria_scope(criteria) + positional = _selector_scope_from_positional(criteria.selectors) + keyword = haskey(criteria, :within) ? criteria.within : nothing + if !isnothing(positional) && !isnothing(keyword) && typeof(positional) != typeof(keyword) + error("Conflicting scope selectors: `$(positional)` and `$(keyword)`.") + end + return isnothing(keyword) ? positional : keyword +end + +function _scope_object_ids(scene::Scene, scope, context) + if isnothing(scope) || scope isa SceneScope + return ObjectId[keys(scene.registry.objects)...] + end + + current_id = _object_id_from_context(context) + if scope isa Self + isnothing(current_id) && error("`Self()` selectors require a current object context.") + return _descendant_ids(scene, current_id) + elseif scope isa SelfPlant + isnothing(current_id) && error("`SelfPlant()` selectors require a current object context.") + plant_id = _ancestor_id(scene, current_id; scale=:Plant) + isnothing(plant_id) && error("No `scale=:Plant` ancestor found for object `$(current_id.value)`.") + return _descendant_ids(scene, plant_id) + elseif scope isa Ancestor + isnothing(current_id) && error("`Ancestor(...)` selectors require a current object context.") + ancestor_id = _ancestor_id(scene, current_id; scale=scope.scale) + if isnothing(ancestor_id) + error("No matching ancestor found for object `$(current_id.value)` and selector `$(scope)`.") + end + return _descendant_ids(scene, ancestor_id) + elseif scope isa Scope + root_id = get(scene.registry.by_name, scope.name, nothing) + if isnothing(root_id) + candidate = ObjectId(scope.name) + root_id = haskey(scene.registry.objects, candidate) ? candidate : nothing + end + isnothing(root_id) && error("No named scope or object `$(scope.name)` found in the scene registry.") + return _descendant_ids(scene, root_id) + end + + error("Unsupported object scope selector `$(scope)` of type `$(typeof(scope))`.") +end + +function _matches_object_criteria(object::Object; scale=nothing, kind=nothing, species=nothing, name=nothing) + isnothing(scale) || object.scale == scale || return false + isnothing(kind) || object.kind == kind || return false + isnothing(species) || object.species == species || return false + isnothing(name) || object.name == name || return false + return true +end + +function resolve_object_ids(scene::Scene, selector::AbstractObjectMultiplicity; context=nothing) + return _resolve_object_ids(scene, selector; context=context) +end + +function _resolve_object_ids( + scene::Scene, + selector::AbstractObjectMultiplicity; + context=nothing, + default_to_context::Bool=false, + default_scope=nothing, +) + criteria_ = criteria(selector) + relation = _criteria_value(criteria_, :relation, Relation) + isnothing(relation) || error("`Relation(...)` selector resolution is not implemented yet.") + + explicit_scope = _criteria_scope(criteria_) + scope = isnothing(explicit_scope) ? default_scope : explicit_scope + scale = _criteria_value(criteria_, :scale, Scale) + kind = _criteria_value(criteria_, :kind, Kind) + species = _criteria_value(criteria_, :species, Species) + name = haskey(criteria_, :name) ? criteria_.name : nothing + + if default_to_context && + isnothing(explicit_scope) && + isnothing(scale) && + isnothing(kind) && + isnothing(species) && + isnothing(name) && + !isnothing(context) + return ObjectId[_object_id_from_context(context)] + end + + candidate_ids = _scope_object_ids(scene, scope, context) + ids = ObjectId[ + id for id in candidate_ids + if _matches_object_criteria(_scene_object(scene, id); scale=scale, kind=kind, species=species, name=name) + ] + _sort_object_ids!(ids) + + if selector isa One && length(ids) != 1 + error("Expected exactly one object for selector `$(selector)`, got $(length(ids)).") + elseif selector isa OptionalOne && length(ids) > 1 + error("Expected zero or one object for selector `$(selector)`, got $(length(ids)).") + end + return ids +end + +resolve_objects(scene::Scene, selector::AbstractObjectMultiplicity; context=nothing) = + [_scene_object(scene, id) for id in resolve_object_ids(scene, selector; context=context)] + +function _default_dependency_scope(scene::Scene, context::ObjectId) + object = _scene_object(scene, context) + (object.scale == :Scene || object.kind == :scene) && return SceneScope() + return Self() +end + +struct CompiledSceneApplication{S,AT,TS,CL,MO} + id::Symbol + spec::S + process::Symbol + name::Union{Nothing,Symbol} + target_ids::Vector{ObjectId} + applies_to::AT + timestep::TS + clock::CL + model_overrides::MO +end + +struct CompiledSceneInputBinding{SEL,P,W,C} + application_id::Symbol + consumer_id::ObjectId + input::Symbol + selector::SEL + origin::Symbol + source_ids::Vector{ObjectId} + source_application_ids::Vector{Symbol} + source_var::Symbol + process::Union{Nothing,Symbol} + application::Union{Nothing,Symbol} + multiplicity::Symbol + policy::P + window::W + carrier_hint::Symbol + carrier::C +end + +struct CompiledSceneCallBinding{SEL} + application_id::Symbol + consumer_id::ObjectId + call::Symbol + selector::SEL + callee_object_ids::Vector{ObjectId} + callee_application_ids::Vector{Symbol} + process::Union{Nothing,Symbol} + application::Union{Nothing,Symbol} + multiplicity::Symbol +end + +struct CompiledEnvironmentBinding{B,C,S} + application_id::Symbol + object_id::ObjectId + provider::Symbol + backend::B + cell::C + required_inputs::Vector{Symbol} + produced_outputs::Vector{Symbol} + support::S + config::Any +end + +struct CompiledEnvironmentBindings{SC,B,I} + scene::SC + bindings::B + by_target::I + scene_revision::Int + environment_revision::Int +end + +struct CompiledScene{SC,AP,AI,IB,CB,IBI,CBI,AO} + scene::SC + applications::AP + applications_by_id::AI + input_bindings::IB + call_bindings::CB + input_bindings_by_target::IBI + call_bindings_by_target::CBI + application_order::AO + revision::Int +end + +function _index_scene_bindings(bindings, application_field::Symbol, object_field::Symbol) + grouped = Dict{Tuple{Symbol,ObjectId},Vector{Any}}() + for binding in bindings + key = ( + getproperty(binding, application_field), + getproperty(binding, object_field), + ) + push!(get!(grouped, key, Any[]), binding) + end + return Dict(key => Tuple(values) for (key, values) in grouped) +end + +function compile_scene(scene::Scene) + return compile_scene(scene, scene.applications) +end + +function compile_scene(scene::Scene, specs::Tuple) + return _compile_scene(scene, specs) +end + +function compile_scene(scene::Scene, specs::AbstractVector) + return _compile_scene(scene, Tuple(specs)) +end + +function compile_scene(scene::Scene, specs...) + return _compile_scene(scene, specs) +end + +function _scene_timeline(scene::Scene) + backend = environment_backend(scene.environment) + try + return _timeline_context(backend) + catch + return TimelineContext(3600.0) + end +end + +function _compile_scene(scene::Scene, raw_specs) + timeline = _scene_timeline(scene) + applications = _compile_scene_applications(scene, raw_specs, timeline) + _validate_scene_writers!(applications) + input_bindings = _compile_scene_input_bindings(scene, applications) + _validate_scene_required_inputs!(scene, applications, input_bindings) + call_bindings = _compile_scene_call_bindings(scene, applications) + input_bindings_by_target = _index_scene_bindings(input_bindings, :application_id, :consumer_id) + call_bindings_by_target = _index_scene_bindings(call_bindings, :application_id, :consumer_id) + application_order = _compile_scene_application_order(applications, input_bindings, call_bindings) + applications_by_id = Dict(application.id => application for application in applications) + return CompiledScene( + scene, + applications, + applications_by_id, + input_bindings, + call_bindings, + input_bindings_by_target, + call_bindings_by_target, + application_order, + scene.revision, + ) +end + +function _compile_scene_applications(scene::Scene, raw_specs, timeline) + process_counts = Dict{Symbol,Int}() + ids = Set{Symbol}() + applications = CompiledSceneApplication[] + for raw_spec in raw_specs + spec = as_model_spec(raw_spec) + selector = applies_to(spec) + isnothing(selector) && error( + "Model application for process `$(process(spec))` has no `AppliesTo(...)` selector." + ) + selector isa AbstractObjectMultiplicity || error( + "`AppliesTo(...)` for process `$(process(spec))` must be an object selector such as `Many(scale=:Leaf)`." + ) + proc = process(spec) + occurrence = get(process_counts, proc, 0) + 1 + process_counts[proc] = occurrence + name = application_name(spec) + app_id = isnothing(name) ? (occurrence == 1 ? proc : Symbol(string(proc), "_", occurrence)) : name + app_id in ids && error("Duplicate compiled scene application id `$(app_id)`.") + push!(ids, app_id) + target_ids = resolve_object_ids(scene, selector) + model_overrides = _compiled_object_model_overrides(spec, target_ids, app_id) + push!( + applications, + CompiledSceneApplication( + app_id, + spec, + proc, + name, + target_ids, + selector, + timestep(spec), + _scene_application_clock(spec, timeline), + model_overrides, + ), + ) + end + return applications +end + +function _compiled_object_model_overrides(spec, target_ids, application_id::Symbol) + model = model_(spec) + model isa ObjectModelOverrides || return nothing + target_set = Set(target_ids) + unmatched = ObjectId[id for id in keys(model.overrides) if !(id in target_set)] + isempty(unmatched) || error( + "Object override(s) `$([id.value for id in unmatched])` for application ", + "`$(application_id)` do not match its `AppliesTo(...)` target set." + ) + return model.overrides +end + +_application_default_model(application::CompiledSceneApplication) = + model_(application.spec) isa ObjectModelOverrides ? + model_(application.spec).base : + model_(application.spec) + +function _application_model(application::CompiledSceneApplication, object_id::ObjectId) + isnothing(application.model_overrides) && return _application_default_model(application) + return get( + application.model_overrides, + object_id, + _application_default_model(application), + ) +end + +function _scene_output_names(application::CompiledSceneApplication) + return Symbol[Symbol(var) for var in keys(outputs_(application.spec))] +end + +function _scene_writer_groups(applications) + groups = Dict{Tuple{ObjectId,Symbol},Vector{Tuple{Int,Any}}}() + for (index, application) in pairs(applications) + for object_id in application.target_ids + for variable in _scene_output_names(application) + push!(get!(groups, (object_id, variable), Tuple{Int,Any}[]), (index, application)) + end + end + end + return groups +end + +function _application_match_labels(application::CompiledSceneApplication) + labels = Set{Symbol}([application.id, application.process]) + isnothing(application.name) || push!(labels, application.name) + return labels +end + +function _update_variables(update) + return Tuple(Symbol(variable) for variable in getproperty(update, :variables)) +end + +function _update_after(update) + return Tuple(Symbol(label) for label in getproperty(update, :after)) +end + +function _matching_updates(spec, variable::Symbol) + return [update for update in updates(spec) if variable in _update_variables(update)] +end + +function _update_after_labels(spec, variable::Symbol) + labels = Symbol[] + for update in _matching_updates(spec, variable) + append!(labels, _update_after(update)) + end + unique!(labels) + return labels +end + +function _updates_after_previous_writer(spec, variable::Symbol, previous_applications) + matching = _matching_updates(spec, variable) + isempty(matching) && return false + previous_labels = Set{Symbol}() + for application in previous_applications + union!(previous_labels, _application_match_labels(application)) + end + for update in matching + after = _update_after(update) + isempty(after) && continue + any(label -> label in previous_labels, after) && return true + end + return false +end + +function _declares_update_without_previous_writer(spec, variable::Symbol, previous_applications) + isempty(previous_applications) || return false + for update in _matching_updates(spec, variable) + isempty(_update_after(update)) || return true + end + return false +end + +function _validate_scene_writers!(applications) + for ((object_id, variable), indexed_writers) in _scene_writer_groups(applications) + length(indexed_writers) <= 1 && continue + sort!(indexed_writers; by=first) + previous = CompiledSceneApplication[] + for (_, application) in indexed_writers + if _declares_update_without_previous_writer(application.spec, variable, previous) + error( + "Application `$(application.id)` declares `Updates($(variable))` for object ", + "`$(object_id.value)`, but no previous writer for `$(variable)` exists. ", + "Move it after the producer named in `after=...`." + ) + end + if !isempty(previous) && !_updates_after_previous_writer(application.spec, variable, previous) + previous_labels = sort!(collect(reduce(union!, (_application_match_labels(app) for app in previous); init=Set{Symbol}()))) + error( + "Variable `$(variable)` on object `$(object_id.value)` is written by multiple ", + "applications. Application `$(application.id)` must declare ", + "`Updates(:$(variable); after=...)` matching one of the previous writers ", + "`$(previous_labels)`." + ) + end + push!(previous, application) + end + end + return nothing +end + +function _scene_application_clock(spec, timeline) + clock = _clock_from_spec_timestep(timestep(spec), timeline) + isnothing(clock) && return ClockSpec(1.0, 0.0) + return clock +end + +function _criteria_get(criteria, key::Symbol, default=nothing) + return haskey(criteria, key) ? getproperty(criteria, key) : default +end + +function _selector_policy(selector::AbstractObjectMultiplicity) + return _criteria_get(criteria(selector), :policy, HoldLast()) +end + +function _selector_window(selector::AbstractObjectMultiplicity) + return _criteria_get(criteria(selector), :window, nothing) +end + +function _selector_var(selector::AbstractObjectMultiplicity, fallback::Symbol) + return _criteria_get(criteria(selector), :var, fallback) +end + +function _selector_application(selector::AbstractObjectMultiplicity) + return _criteria_get(criteria(selector), :application, nothing) +end + +function _dependency_object_ids(scene::Scene, selector::AbstractObjectMultiplicity, context::ObjectId) + return _resolve_object_ids( + scene, + selector; + context=context, + default_to_context=true, + default_scope=_default_dependency_scope(scene, context), + ) +end + +function _carrier_hint(selector::AbstractObjectMultiplicity, policy, window) + !isnothing(window) && return :temporal_stream + policy isa HoldLast || return :temporal_stream + selector isa Many && return :ref_vector + return :shared_ref +end + +function _status_ref_or_nothing(status, var::Symbol) + status isa Status || return nothing + var in propertynames(status) || return nothing + return refvalue(status, var) +end + +function _input_carrier(scene::Scene, selector::AbstractObjectMultiplicity, source_ids::Vector{ObjectId}, source_var::Symbol) + refs = Base.RefValue[] + for source_id in source_ids + object = _scene_object(scene, source_id) + source_ref = _status_ref_or_nothing(object.status, source_var) + isnothing(source_ref) && return nothing + push!(refs, source_ref) + end + if selector isa Many + isempty(refs) && return RefVector{Any}() + return _ref_vector_carrier(refs) + end + return isempty(refs) ? nothing : only(refs) +end + +function _ref_vector_carrier(refs) + T = typeof(refs[1][]) + typed_refs = Base.RefValue{T}[] + for source_ref in refs + source_ref isa Base.RefValue{T} || return ObjectRefVector(refs) + push!(typed_refs, source_ref) + end + return RefVector(typed_refs) +end + +input_carrier(binding::CompiledSceneInputBinding) = binding.carrier +has_reference_carrier(binding::CompiledSceneInputBinding) = !isnothing(binding.carrier) +input_value(binding::CompiledSceneInputBinding) = _input_value(binding.carrier) +_input_value(::Nothing) = nothing +_input_value(carrier::Base.RefValue) = carrier[] +_input_value(carrier::RefVector) = carrier +_input_value(carrier::ObjectRefVector) = carrier + +function _matching_input_source_applications(applications_by_object, source_ids, source_var::Symbol, process_filter, application_filter) + matches = Symbol[] + for source_id in source_ids + for application in get(applications_by_object, source_id, Any[]) + source_var in _scene_output_names(application) || continue + isnothing(process_filter) || application.process == process_filter || continue + isnothing(application_filter) || application.id == application_filter || continue + push!(matches, application.id) + end + end + unique!(matches) + if (!isnothing(process_filter) || !isnothing(application_filter)) && isempty(matches) + error( + "Input selector for source variable `$(source_var)` requested", + isnothing(process_filter) ? "" : " process `$(process_filter)`", + isnothing(application_filter) ? "" : " application `$(application_filter)`", + ", but no matching source application was found." + ) + end + return matches +end + +function _compile_scene_input_bindings(scene::Scene, applications) + bindings = CompiledSceneInputBinding[] + by_object = _applications_by_object(applications) + for application in applications + for consumer_id in application.target_ids + declared_inputs = value_inputs(application.spec) + declared_inputs isa NamedTuple || (declared_inputs = NamedTuple()) + for (input_name, selector) in pairs(declared_inputs) + input_sym = Symbol(input_name) + _validate_declared_scene_input_name!(application, input_sym) + selector isa AbstractObjectMultiplicity || error( + "Input binding `$(input_sym)` on application `$(application.id)` must use an object selector." + ) + _push_scene_input_binding!( + bindings, + scene, + application, + consumer_id, + input_sym, + selector, + :declared, + by_object, + ) + end + _append_inferred_scene_input_bindings!(bindings, scene, application, consumer_id, declared_inputs, by_object) + end + end + return bindings +end + +function _push_scene_input_binding!( + bindings, + scene::Scene, + application::CompiledSceneApplication, + consumer_id::ObjectId, + input_sym::Symbol, + selector::AbstractObjectMultiplicity, + origin::Symbol, + applications_by_object, + source_ids_override=nothing, +) + source_ids = isnothing(source_ids_override) ? _dependency_object_ids(scene, selector, consumer_id) : source_ids_override + isempty(source_ids) && selector isa OptionalOne && return bindings + policy = _selector_policy(selector) + window = _selector_window(selector) + source_var = _selector_var(selector, input_sym) + process_filter = _criteria_get(criteria(selector), :process, nothing) + application_filter = _selector_application(selector) + source_application_ids = _matching_input_source_applications( + applications_by_object, + source_ids, + source_var, + process_filter, + application_filter, + ) + carrier = _input_carrier(scene, selector, source_ids, source_var) + carrier_hint = _carrier_hint(selector, policy, window) + _validate_scene_input_source!( + scene, + application, + consumer_id, + input_sym, + source_var, + source_ids, + carrier, + carrier_hint, + ) + push!( + bindings, + CompiledSceneInputBinding( + application.id, + consumer_id, + input_sym, + selector, + origin, + source_ids, + source_application_ids, + source_var, + process_filter, + application_filter, + multiplicity(selector), + policy, + window, + carrier_hint, + carrier, + ), + ) + return bindings +end + +function _scene_input_names(application::CompiledSceneApplication) + return Symbol[Symbol(var) for var in keys(inputs_(application.spec))] +end + +function _validate_scene_input_source!( + scene::Scene, + application::CompiledSceneApplication, + consumer_id::ObjectId, + input_sym::Symbol, + source_var::Symbol, + source_ids::Vector{ObjectId}, + carrier, + carrier_hint::Symbol, +) + carrier_hint == :temporal_stream && return nothing + !isnothing(carrier) && return nothing + status_source_ids = ObjectId[ + source_id for source_id in source_ids + if _scene_object(scene, source_id).status isa Status + ] + isempty(status_source_ids) && return nothing + error( + "Input binding `$(input_sym)` on application `$(application.id)` for object ", + "`$(consumer_id.value)` reads `$(source_var)` from objects ", + "`$([id.value for id in status_source_ids])`, but no source `Status` reference is available." + ) +end + +function _validate_declared_scene_input_name!(application::CompiledSceneApplication, input_sym::Symbol) + input_names = Set(_scene_input_names(application)) + input_sym in input_names && return nothing + error( + "Input binding `$(input_sym)` on application `$(application.id)` is not declared by ", + "`inputs_` for process `$(application.process)`. Declared model inputs are ", + "`$(sort!(collect(input_names)))`." + ) +end + +function _same_object_output_applications(applications_by_object, application::CompiledSceneApplication, object_id::ObjectId, variable::Symbol) + matches = CompiledSceneApplication[] + for candidate in get(applications_by_object, object_id, Any[]) + candidate.id == application.id && continue + variable in _scene_output_names(candidate) || continue + push!(matches, candidate) + end + return matches +end + +function _append_inferred_scene_input_bindings!( + bindings, + scene::Scene, + application::CompiledSceneApplication, + consumer_id::ObjectId, + declared_inputs, + applications_by_object, +) + declared_names = declared_inputs isa NamedTuple ? Set(Symbol.(keys(declared_inputs))) : Set{Symbol}() + for input_sym in _scene_input_names(application) + input_sym in declared_names && continue + matches = _same_object_output_applications(applications_by_object, application, consumer_id, input_sym) + isempty(matches) && continue + if length(matches) > 1 + error( + "Input `$(input_sym)` on application `$(application.id)` for object `$(consumer_id.value)` ", + "has ambiguous same-object producers: `$([match.id for match in matches])`. ", + "Add `Inputs(:$(input_sym) => One(...))` to disambiguate." + ) + end + producer = only(matches) + selector = One(within=Self(), process=producer.process, application=producer.id, var=input_sym) + _push_scene_input_binding!( + bindings, + scene, + application, + consumer_id, + input_sym, + selector, + :inferred_same_object, + applications_by_object, + ObjectId[consumer_id], + ) + end + return bindings +end + +function _bound_scene_inputs(input_bindings) + bound = Set{Tuple{Symbol,ObjectId,Symbol}}() + for binding in input_bindings + push!(bound, (binding.application_id, binding.consumer_id, binding.input)) + end + return bound +end + +function _status_has_variable(scene::Scene, object_id::ObjectId, variable::Symbol) + object = _scene_object(scene, object_id) + object.status isa Status || return false + return variable in propertynames(object.status) +end + +function _validate_scene_required_inputs!(scene::Scene, applications, input_bindings) + bound = _bound_scene_inputs(input_bindings) + missing = NamedTuple[] + for application in applications + for object_id in application.target_ids + for input in _scene_input_names(application) + (application.id, object_id, input) in bound && continue + _status_has_variable(scene, object_id, input) && continue + push!( + missing, + ( + application_id=application.id, + object_id=object_id.value, + input=input, + process=application.process, + ), + ) + end + end + end + isempty(missing) && return nothing + details = join( + [ + "`$(row.application_id)` on object `$(row.object_id)` requires `$(row.input)`" + for row in missing + ], + "; ", + ) + error( + "Missing required scene/object input(s): ", + details, + ". Provide the variable on object `Status`, add an `Inputs(...)` binding, ", + "or add an unambiguous same-object producer." + ) +end + +function _applications_by_object(applications) + by_object = Dict{ObjectId,Vector{Any}}() + for application in applications + for object_id in application.target_ids + push!(get!(by_object, object_id, Any[]), application) + end + end + return by_object +end + +function _matching_callee_applications(applications, object_id::ObjectId, proc, application_name_filter) + matches = Symbol[] + for application in get(applications, object_id, Any[]) + isnothing(proc) || application.process == proc || continue + isnothing(application_name_filter) || application.id == application_name_filter || continue + push!(matches, application.id) + end + return matches +end + +function _compile_scene_call_bindings(scene::Scene, applications) + by_object = _applications_by_object(applications) + bindings = CompiledSceneCallBinding[] + for application in applications + calls = model_calls(application.spec) + calls isa NamedTuple || continue + for consumer_id in application.target_ids + for (call_name, selector) in pairs(calls) + call_sym = Symbol(call_name) + selector isa AbstractObjectMultiplicity || error( + "Call binding `$(call_sym)` on application `$(application.id)` must use an object selector." + ) + callee_object_ids = _dependency_object_ids(scene, selector, consumer_id) + proc = _criteria_get(criteria(selector), :process, nothing) + app_name = _selector_application(selector) + callee_application_ids = Symbol[] + for object_id in callee_object_ids + append!( + callee_application_ids, + _matching_callee_applications(by_object, object_id, proc, app_name), + ) + end + unique!(callee_application_ids) + if isempty(callee_application_ids) + error( + "Call `$(call_sym)` on application `$(application.id)` matched objects ", + "$([id.value for id in callee_object_ids]) but no model application", + isnothing(proc) ? "." : " with process `$(proc)`.", + ) + end + if selector isa One && length(callee_application_ids) != 1 + error( + "Call `$(call_sym)` on application `$(application.id)` expected one callee application, ", + "got `$(callee_application_ids)`. Add `application=:name` to disambiguate." + ) + elseif selector isa OptionalOne && length(callee_application_ids) > 1 + error( + "Call `$(call_sym)` on application `$(application.id)` expected zero or one callee application, ", + "got `$(callee_application_ids)`. Add `application=:name` to disambiguate." + ) + end + push!( + bindings, + CompiledSceneCallBinding( + application.id, + consumer_id, + call_sym, + selector, + callee_object_ids, + callee_application_ids, + proc, + app_name, + multiplicity(selector), + ), + ) + end + end + end + return bindings +end + +function _scene_call_owners(call_bindings) + owners = Dict{Symbol,Set{Symbol}}() + for binding in call_bindings + for callee_id in binding.callee_application_ids + push!(get!(owners, callee_id, Set{Symbol}()), binding.application_id) + end + end + return owners +end + +function _add_scene_application_edge!(children, parent::Symbol, child::Symbol) + parent == child && return nothing + push!(get!(children, parent, Set{Symbol}()), child) + return nothing +end + +function _scene_input_order_edges!(children, input_bindings, call_owners) + for binding in input_bindings + for source_id in binding.source_application_ids + owners = get(call_owners, source_id, nothing) + if isnothing(owners) + _add_scene_application_edge!(children, source_id, binding.application_id) + else + for owner_id in owners + _add_scene_application_edge!(children, owner_id, binding.application_id) + end + end + end + end + return children +end + +function _scene_update_order_edges!(children, applications) + for indexed_writers in values(_scene_writer_groups(applications)) + length(indexed_writers) > 1 || continue + sort!(indexed_writers; by=first) + for index in 2:length(indexed_writers) + previous_application = indexed_writers[index - 1][2] + application = indexed_writers[index][2] + _add_scene_application_edge!(children, previous_application.id, application.id) + end + end + return children +end + +function _stable_topological_application_order(applications, children) + application_ids = Symbol[application.id for application in applications] + positions = Dict(application_id => index for (index, application_id) in pairs(application_ids)) + indegree = Dict(application_id => 0 for application_id in application_ids) + for child_ids in values(children) + for child_id in child_ids + indegree[child_id] = get(indegree, child_id, 0) + 1 + end + end + ready = Symbol[application_id for application_id in application_ids if indegree[application_id] == 0] + order = Symbol[] + while !isempty(ready) + sort!(ready; by=application_id -> positions[application_id]) + application_id = popfirst!(ready) + push!(order, application_id) + child_ids = sort!(collect(get(children, application_id, Set{Symbol}())); by=child_id -> positions[child_id]) + for child_id in child_ids + indegree[child_id] -= 1 + indegree[child_id] == 0 && push!(ready, child_id) + end + end + if length(order) != length(application_ids) + remaining = Symbol[application_id for application_id in application_ids if indegree[application_id] > 0] + error( + "Scene application dependency cycle detected among applications `$(remaining)`. ", + "Break the same-timestep cycle with a temporal policy or revise `Inputs(...)`/`Updates(...)`." + ) + end + return order +end + +function _compile_scene_application_order(applications, input_bindings, call_bindings) + children = Dict{Symbol,Set{Symbol}}() + call_owners = _scene_call_owners(call_bindings) + _scene_input_order_edges!(children, input_bindings, call_owners) + _scene_update_order_edges!(children, applications) + return _stable_topological_application_order(applications, children) +end + +function _ordered_scene_applications(compiled::CompiledScene) + return [compiled.applications_by_id[application_id] for application_id in compiled.application_order] +end + +function explain_scene_applications(compiled::CompiledScene) + return [ + ( + application_id=application.id, + process=application.process, + name=application.name, + target_ids=[id.value for id in application.target_ids], + applies_to=application.applies_to, + timestep=application.timestep, + clock=application.clock, + model_type=typeof(_application_default_model(application)), + model_storage=isnothing(application.model_overrides) ? :shared_application : :per_object_override, + model_dispatch=_application_model_dispatch(application), + object_overrides=isnothing(application.model_overrides) ? + NamedTuple[] : + [ + ( + object_id=object_id.value, + model_type=typeof(model), + ) + for (object_id, model) in sort!( + collect(application.model_overrides); + by=pair -> string(first(pair).value), + ) + ], + ) + for application in compiled.applications + ] +end + +function _application_model_dispatch(application::CompiledSceneApplication) + isnothing(application.model_overrides) && return :concrete_shared + override_type = valtype(typeof(application.model_overrides)) + default_type = typeof(_application_default_model(application)) + return isconcretetype(override_type) && default_type == override_type ? + :concrete_per_object : + :heterogeneous_per_object +end + +function explain_schedule(compiled::CompiledScene) + timeline = _scene_timeline(compiled.scene) + manual_application_ids = _manual_call_application_ids(compiled) + execution_positions = Dict(application_id => index for (index, application_id) in pairs(compiled.application_order)) + return [ + ( + application_id=application.id, + process=application.process, + execution_index=execution_positions[application.id], + timestep=application.timestep, + clock=application.clock, + dt_steps=application.clock.dt, + phase=application.clock.phase, + dt_seconds=float(application.clock.dt) * timeline.base_step_seconds, + target_ids=[id.value for id in application.target_ids], + root_scheduled=!(application.id in manual_application_ids), + manual_call_only=application.id in manual_application_ids, + ) + for application in _ordered_scene_applications(compiled) + ] +end + +function _scene_binding_carrier_kind(binding::CompiledSceneInputBinding) + binding.carrier_hint == :temporal_stream && return :temporal_stream + carrier = binding.carrier + isnothing(carrier) && return :unresolved + carrier isa Base.RefValue && return :ref + carrier isa RefVector && return :ref_vector + carrier isa ObjectRefVector && return :object_ref_vector + return :custom +end + +function _scene_binding_copy_semantics(binding::CompiledSceneInputBinding) + kind = _scene_binding_carrier_kind(binding) + kind in (:ref, :ref_vector, :object_ref_vector) && return :live_references + kind == :temporal_stream && return :materialized_temporal_value + kind == :unresolved && return :not_materialized + return :backend_defined +end + +function explain_bindings(compiled::CompiledScene) + return [ + ( + application_id=binding.application_id, + consumer_id=binding.consumer_id.value, + input=binding.input, + origin=binding.origin, + source_ids=[id.value for id in binding.source_ids], + source_application_ids=binding.source_application_ids, + source_var=binding.source_var, + process=binding.process, + application=binding.application, + multiplicity=binding.multiplicity, + policy=binding.policy, + window=binding.window, + carrier_hint=binding.carrier_hint, + carrier_kind=_scene_binding_carrier_kind(binding), + copy_semantics=_scene_binding_copy_semantics(binding), + has_reference_carrier=has_reference_carrier(binding), + carrier_type=isnothing(binding.carrier) ? nothing : typeof(binding.carrier), + selector=binding.selector, + ) + for binding in compiled.input_bindings + ] +end + +function explain_calls(compiled::CompiledScene) + return [ + ( + application_id=binding.application_id, + consumer_id=binding.consumer_id.value, + call=binding.call, + callee_object_ids=[id.value for id in binding.callee_object_ids], + callee_application_ids=binding.callee_application_ids, + process=binding.process, + application=binding.application, + multiplicity=binding.multiplicity, + selector=binding.selector, + ) + for binding in compiled.call_bindings + ] +end + +function explain_writers(compiled::CompiledScene) + groups = _scene_writer_groups(compiled.applications) + rows = NamedTuple[] + for ((object_id, variable), indexed_writers) in groups + sort!(indexed_writers; by=first) + applications = [application for (_, application) in indexed_writers] + push!( + rows, + ( + object_id=object_id.value, + variable=variable, + application_ids=[application.id for application in applications], + processes=[application.process for application in applications], + update_application_ids=[ + application.id for application in applications + if !isempty(_matching_updates(application.spec, variable)) + ], + update_after=[ + application.id => _update_after_labels(application.spec, variable) + for application in applications + if !isempty(_matching_updates(application.spec, variable)) + ], + duplicate=length(applications) > 1, + ), + ) + end + sort!(rows; by=row -> (string(row.object_id), string(row.variable))) + return rows +end + +function _environment_config_payload(config) + config isa EnvironmentConfig && return config.config + return config +end + +function _environment_backend_from_config(scene::Scene, config) + payload = _environment_config_payload(config) + isnothing(payload) && return environment_backend(scene.environment) + payload isa NamedTuple && haskey(payload, :backend) && return environment_backend(payload.backend) + payload isa AbstractEnvironmentBackend && return payload + return environment_backend(scene.environment) +end + +function _environment_provider_from_config(config, backend) + payload = _environment_config_payload(config) + payload isa NamedTuple && haskey(payload, :provider) && return Symbol(payload.provider) + payload isa Symbol && return payload + isnothing(backend) && return :none + return :scene +end + +function _object_environment_support(application::CompiledSceneApplication, object::Object) + scale = isnothing(object.scale) ? :Default : object.scale + return EnvironmentSupport(application.id, scale, application.process, object.status) +end + +bind_environment(backend, object::Object, support, config=nothing) = :global + +function _scene_environment_entities(scene::Scene) + return [ + ( + id=object.id.value, + object=object, + scale=object.scale, + kind=object.kind, + species=object.species, + name=object.name, + parent=isnothing(object.parent) ? nothing : object.parent.value, + geometry=geometry(object), + position=position(object), + bounds=bounds(object), + status=object.status, + ) + for object in scene_objects(scene) + ] +end + +function _scene_environment_backends(scene::Scene, compiled::CompiledScene) + backends = Any[] + seen = Set{UInt}() + for application in compiled.applications + backend = _environment_backend_from_config(scene, environment_config(application.spec)) + isnothing(backend) && continue + id = objectid(backend) + id in seen && continue + push!(seen, id) + push!(backends, backend) + end + return backends +end + +function _update_scene_environment_indices!(scene::Scene, compiled::CompiledScene) + entities = _scene_environment_entities(scene) + for backend in _scene_environment_backends(scene, compiled) + update_index!(backend, entities) + end + return nothing +end + +function _environment_variable_names(vars) + return Symbol[Symbol(var) for var in keys(vars)] +end + +function _compile_environment_bindings(scene::Scene, compiled::CompiledScene) + bindings = CompiledEnvironmentBinding[] + for application in compiled.applications + config = environment_config(application.spec) + backend = _environment_backend_from_config(scene, config) + provider = _environment_provider_from_config(config, backend) + required_inputs = _environment_variable_names(meteo_inputs_(application.spec)) + produced_outputs = _environment_variable_names(meteo_outputs_(application.spec)) + for object_id in application.target_ids + object = _scene_object(scene, object_id) + support = _object_environment_support(application, object) + cell = bind_environment(backend, object, support, _environment_config_payload(config)) + push!( + bindings, + CompiledEnvironmentBinding( + application.id, + object_id, + provider, + backend, + cell, + required_inputs, + produced_outputs, + support, + config, + ), + ) + end + end + return bindings +end + +function compile_environment_bindings(scene::Scene, compiled::CompiledScene=refresh_bindings!(scene)) + _update_scene_environment_indices!(scene, compiled) + bindings = _compile_environment_bindings(scene, compiled) + by_target = Dict( + (binding.application_id, binding.object_id) => binding + for binding in bindings + ) + length(by_target) == length(bindings) || error( + "Environment binding compilation produced duplicate `(application_id, object_id)` targets." + ) + return CompiledEnvironmentBindings( + scene, + bindings, + by_target, + scene.revision, + scene.environment_revision, + ) +end + +function explain_environment_bindings(compiled::CompiledEnvironmentBindings) + return [ + ( + application_id=binding.application_id, + object_id=binding.object_id.value, + provider=binding.provider, + backend_type=isnothing(binding.backend) ? nothing : typeof(binding.backend), + cell=binding.cell, + required_inputs=binding.required_inputs, + produced_outputs=binding.produced_outputs, + support=binding.support, + config=binding.config, + ) + for binding in compiled.bindings + ] +end + +function explain_environment_bindings(scene::Scene) + return explain_environment_bindings(refresh_environment_bindings!(scene)) +end + +struct SceneRunContext{CS,EB,A,TS,C} + compiled::CS + environment_bindings::EB + application::A + object_id::ObjectId + temporal_streams::TS + time::Float64 + constants::C +end + +struct SceneCallTarget{CS,EB,A,S,TS,C} + compiled::CS + environment_bindings::EB + application::A + object_id::ObjectId + model + status::S + temporal_streams::TS + time::Float64 + constants::C +end + +function _compiled_application_by_id(compiled::CompiledScene, id::Symbol) + application = get(compiled.applications_by_id, id, nothing) + isnothing(application) && error("No compiled scene application with id `$(id)`.") + return application +end + +function _environment_binding_for(env_bindings::CompiledEnvironmentBindings, application_id::Symbol, object_id::ObjectId) + return get(env_bindings.by_target, (application_id, object_id), nothing) +end + +function _scene_object_status(scene::Scene, object_id::ObjectId) + object = _scene_object(scene, object_id) + object.status isa Status || error( + "Scene object `$(object_id.value)` has no `Status`; scene runtime requires status-backed objects." + ) + return object.status +end + +function _set_status_if_present!(status::Status, variable::Symbol, value) + variable in propertynames(status) || error( + "Cannot materialize input `$(variable)` because consumer status has no such variable." + ) + status[variable] = value + return status +end + +_scene_stream_key(object_id::ObjectId, variable::Symbol) = (object_id, variable) + +function _scene_publish_outputs!(streams, application::CompiledSceneApplication, object_id::ObjectId, status, time::Real) + isnothing(streams) && return nothing + for variable in keys(outputs_(application.spec)) + var = Symbol(variable) + hasproperty(status, var) || error( + "Application `$(application.id)` declares output `$(var)`, but object ", + "`$(object_id.value)` status has no such variable." + ) + key = _scene_stream_key(object_id, var) + samples = get!(streams, key, Tuple{Float64,Any}[]) + filter!(sample -> !isapprox(sample[1], float(time); atol=1.0e-8, rtol=0.0), samples) + push!(samples, (float(time), getproperty(status, var))) + end + return nothing +end + +function _scene_latest_sample(samples, time::Real) + latest = nothing + latest_t = -Inf + for (sample_t, value) in samples + sample_t <= float(time) || continue + sample_t >= latest_t || continue + latest = value + latest_t = sample_t + end + return latest +end + +function _scene_window_samples(samples, t_start::Real, t_end::Real) + return Any[value for (sample_t, value) in samples if float(t_start) <= sample_t <= float(t_end)] +end + +function _scene_window_reduce(values, durations, policy) + isempty(values) && return 0.0 + reducer = policy isa Integrate ? policy.reducer : (policy isa Aggregate ? policy.reducer : PlantMeteo.SumReducer()) + f = _normalize_policy_reducer(reducer) + applicable(f, values, durations) && return f(values, durations) + applicable(f, values) && return f(values) + reducer isa PlantMeteo.SumReducer && return sum(values) + reducer isa PlantMeteo.MeanReducer && return Statistics.mean(values) + reducer isa PlantMeteo.MinReducer && return minimum(values) + reducer isa PlantMeteo.MaxReducer && return maximum(values) + reducer isa PlantMeteo.FirstReducer && return first(values) + reducer isa PlantMeteo.LastReducer && return last(values) + error( + "Reducer `$(reducer)` is not callable on scene temporal input values for policy ", + "`$(typeof(policy))`. Expected `(values)` or `(values, durations_seconds)`." + ) +end + +function _scene_duration_steps(duration, timeline) + if duration isa Dates.Period || duration isa Dates.CompoundPeriod || duration isa Real + seconds = _duration_to_seconds(duration) + isnothing(seconds) && return nothing + steps = seconds / timeline.base_step_seconds + steps >= 1.0 || error( + "Input window `$(duration)` is shorter than the scene base step ", + "($(timeline.base_step_seconds) seconds)." + ) + return steps + end + return nothing +end + +function _scene_input_window_steps(binding::CompiledSceneInputBinding, application::CompiledSceneApplication, timeline) + explicit = _scene_duration_steps(binding.window, timeline) + !isnothing(explicit) && return explicit + binding.policy isa Union{Integrate,Aggregate} && return float(application.clock.dt) + return 1.0 +end + +function _scene_temporal_source_value(streams, source_id::ObjectId, source_var::Symbol, time::Real, policy, t_start::Real, timeline) + samples = get(streams, _scene_stream_key(source_id, source_var), nothing) + isnothing(samples) && return policy isa Union{Integrate,Aggregate} ? 0.0 : nothing + if policy isa HoldLast + return _scene_latest_sample(samples, time) + elseif policy isa Union{Integrate,Aggregate} + values = _scene_window_samples(samples, t_start, time) + durations = fill(timeline.base_step_seconds, length(values)) + return _scene_window_reduce(values, durations, policy) + elseif policy isa Interpolate + return _scene_latest_sample(samples, time) + end + error("Unsupported scene temporal input policy `$(typeof(policy))`.") +end + +function _scene_temporal_input_value(binding::CompiledSceneInputBinding, application::CompiledSceneApplication, streams, time::Real, timeline) + window_steps = _scene_input_window_steps(binding, application, timeline) + t_start = float(time) - float(window_steps) + 1.0 + if binding.multiplicity == :many + return [ + _scene_temporal_source_value(streams, source_id, binding.source_var, time, binding.policy, t_start, timeline) + for source_id in binding.source_ids + ] + end + source_id = only(binding.source_ids) + value = _scene_temporal_source_value(streams, source_id, binding.source_var, time, binding.policy, t_start, timeline) + isnothing(value) && error( + "No temporal scene value available for input `$(binding.input)` from ", + "`$(source_id.value).$(binding.source_var)` at t=$(time)." + ) + return value +end + +function _scene_assign_input_value!(status::Status, variable::Symbol, value) + variable in propertynames(status) || error( + "Cannot materialize input `$(variable)` because consumer status has no such variable." + ) + current = status[variable] + if current isa RefVector && value isa AbstractVector + length(current) != length(value) && resize!(current, length(value)) + for i in eachindex(value) + current[i] = value[i] + end + return status + end + status[variable] = value + return status +end + +function _materialize_scene_inputs!( + compiled::CompiledScene, + application::CompiledSceneApplication, + object_id::ObjectId, + streams=nothing, + time::Real=1, +) + status = _scene_object_status(compiled.scene, object_id) + timeline = _scene_timeline(compiled.scene) + bindings = get(compiled.input_bindings_by_target, (application.id, object_id), ()) + for binding in bindings + if binding.carrier_hint == :temporal_stream + isnothing(streams) && continue + value = _scene_temporal_input_value(binding, application, streams, time, timeline) + _scene_assign_input_value!(status, binding.input, value) + elseif has_reference_carrier(binding) + _set_status_if_present!(status, binding.input, input_value(binding)) + end + end + return status +end + +function _scene_meteo_for_model( + env_bindings::CompiledEnvironmentBindings, + application::CompiledSceneApplication, + object_id::ObjectId, + time::Real, +) + binding = _environment_binding_for(env_bindings, application.id, object_id) + isnothing(binding) && return nothing + isnothing(binding.backend) && return nothing + return sample_environment(binding.backend, binding.support, time, application.spec) +end + +function _scatter_scene_environment_outputs!( + env_bindings::CompiledEnvironmentBindings, + application::CompiledSceneApplication, + object_id::ObjectId, + status, + time::Real, +) + isempty(keys(meteo_outputs_(application.spec))) && return nothing + binding = _environment_binding_for(env_bindings, application.id, object_id) + isnothing(binding) && return nothing + isnothing(binding.backend) && return nothing + return scatter_environment_outputs!(binding.backend, binding.support, time, application.spec, status) +end + +function _run_scene_application!( + compiled::CompiledScene, + env_bindings::CompiledEnvironmentBindings, + application::CompiledSceneApplication, + object_id::ObjectId; + time::Real=1, + constants=nothing, + temporal_streams=nothing, + publish::Bool=true, +) + status = _materialize_scene_inputs!(compiled, application, object_id, temporal_streams, time) + meteo = _scene_meteo_for_model(env_bindings, application, object_id, time) + context = SceneRunContext(compiled, env_bindings, application, object_id, temporal_streams, float(time), constants) + model = _application_model(application, object_id) + run!(model, nothing, status, meteo, constants, context) + if publish + _scatter_scene_environment_outputs!(env_bindings, application, object_id, status, time) + _scene_publish_outputs!(temporal_streams, application, object_id, status, time) + end + return status +end + +_scene_application_should_run(application::CompiledSceneApplication, t::Real) = + _should_run_at_time(application.clock, float(t)) + +function _manual_call_application_ids(compiled::CompiledScene) + ids = Set{Symbol}() + for binding in compiled.call_bindings + union!(ids, binding.callee_application_ids) + end + return ids +end + +function _scene_call_targets(context::SceneRunContext, name::Symbol) + targets = SceneCallTarget[] + bindings = get( + context.compiled.call_bindings_by_target, + (context.application.id, context.object_id), + (), + ) + for binding in bindings + binding.call == name || continue + for application_id in binding.callee_application_ids + callee_application = _compiled_application_by_id(context.compiled, application_id) + for object_id in binding.callee_object_ids + object_id in callee_application.target_ids || continue + status = _scene_object_status(context.compiled.scene, object_id) + push!( + targets, + SceneCallTarget( + context.compiled, + context.environment_bindings, + callee_application, + object_id, + _application_model(callee_application, object_id), + status, + context.temporal_streams, + context.time, + context.constants, + ), + ) + end + end + end + return targets +end + +dependency_targets(context::SceneRunContext, name::Symbol) = _scene_call_targets(context, name) +dependency_target(context::SceneRunContext, name::Symbol) = only(dependency_targets(context, name)) + +function run_call!(target::SceneCallTarget; publish::Bool=true) + _run_scene_application!( + target.compiled, + target.environment_bindings, + target.application, + target.object_id; + time=target.time, + constants=target.constants, + temporal_streams=target.temporal_streams, + publish=publish, + ) + return target +end + +function run!(scene::Scene; steps::Integer=1, constants=nothing) + compiled = refresh_bindings!(scene) + env_bindings = refresh_environment_bindings!(scene, compiled) + manual_application_ids = _manual_call_application_ids(compiled) + temporal_streams = Dict{Tuple{ObjectId,Symbol},Vector{Tuple{Float64,Any}}}() + ordered_applications = _ordered_scene_applications(compiled) + for step in 1:steps + for application in ordered_applications + application.id in manual_application_ids && continue + _scene_application_should_run(application, step) || continue + for object_id in application.target_ids + _run_scene_application!( + compiled, + env_bindings, + application, + object_id; + time=step, + constants=constants, + temporal_streams=temporal_streams, + publish=true, + ) + end + end + end + return scene +end + +struct ObjectAddress{SC,K,SP,S,N,P,V,R,M} + scope::SC + kind::K + species::SP + scale::S + name::N + process::P + var::V + relation::R + multiplicity::M +end + +function ObjectAddress(selector::AbstractObjectMultiplicity) + c = criteria(selector) + scope = haskey(c, :within) ? c.within : nothing + kind = haskey(c, :kind) ? c.kind : nothing + species = haskey(c, :species) ? c.species : nothing + scale = haskey(c, :scale) ? c.scale : nothing + name = haskey(c, :name) ? c.name : nothing + process = haskey(c, :process) ? c.process : nothing + var = haskey(c, :var) ? c.var : nothing + relation = haskey(c, :relation) ? c.relation : nothing + return ObjectAddress(scope, kind, species, scale, name, process, var, relation, multiplicity(selector)) +end + +object_address(selector::AbstractObjectMultiplicity) = ObjectAddress(selector) + +struct Input{S} + selector::S +end +Input(; kwargs...) = Input(One(; kwargs...)) + +struct Call{S} + selector::S +end +Call(; kwargs...) = Call(One(; kwargs...)) + +struct EnvironmentConfig{C} + config::C +end + +_normalize_application_name(name) = isnothing(name) ? nothing : Symbol(name) + +function _normalize_application_bindings(bindings::NamedTuple) + return bindings +end + +function _normalize_application_bindings(bindings::Tuple) + pairs = Pair{Symbol,Any}[] + for binding in bindings + binding isa Pair || error( + "Expected `var => selector` pairs in `Inputs(...)` or `Calls(...)`, got `$(typeof(binding))`." + ) + first(binding) isa Union{Symbol,AbstractString} || error( + "Binding names in `Inputs(...)` and `Calls(...)` must be symbols or strings." + ) + push!(pairs, Symbol(first(binding)) => last(binding)) + end + return (; pairs...) +end + +function _normalize_application_bindings(binding::Pair) + return _normalize_application_bindings((binding,)) +end + +function _normalize_application_bindings(bindings) + error( + "Unsupported binding declaration `$(bindings)` of type `$(typeof(bindings))`. ", + "Use pairs such as `:x => Many(...)` or keyword arguments." + ) +end + +function _model_default_value_inputs(model) + defaults = Pair{Symbol,Any}[] + for (dep_name, selector) in pairs(dep(model)) + selector isa Input || continue + push!(defaults, Symbol(dep_name) => selector.selector) + end + return (; defaults...) +end + +function _model_default_model_calls(model) + defaults = Pair{Symbol,Any}[] + for (dep_name, selector) in pairs(dep(model)) + selector isa Call || continue + push!(defaults, Symbol(dep_name) => selector.selector) + end + return (; defaults...) +end + +function _merge_value_inputs(defaults::NamedTuple, explicit::NamedTuple) + return (; pairs(defaults)..., pairs(explicit)...) +end + +function _legacy_multiscale_rhs_from_input_selector(selector::AbstractObjectMultiplicity) + c = criteria(selector) + haskey(c, :scale) || return nothing + + # The current MTG mapping layer only understands scale/variable mappings. + # Keep richer object filters as unified metadata for the future compiler. + unsupported = (:kind, :domain, :species, :name, :process, :relation) + any(key -> haskey(c, key) && !isnothing(getproperty(c, key)), unsupported) && return nothing + + scale = c.scale + src_var = haskey(c, :var) ? c.var : nothing + if selector isa Many + return isnothing(src_var) ? [scale] : [scale => src_var] + elseif selector isa One || selector isa OptionalOne + return isnothing(src_var) ? scale : (scale => src_var) + end + return nothing +end + +function _legacy_multiscale_from_value_inputs(bindings::NamedTuple, model=nothing) + mapped = Pair{Symbol,Any}[] + model_input_names = isnothing(model) ? nothing : Set(keys(inputs_(model))) + for (input_var, selector) in pairs(bindings) + input_sym = Symbol(input_var) + if !isnothing(model_input_names) && !(input_sym in model_input_names) + continue + end + selector isa AbstractObjectMultiplicity || continue + rhs = _legacy_multiscale_rhs_from_input_selector(selector) + isnothing(rhs) && continue + push!(mapped, input_sym => rhs) + end + return mapped +end + +function _merge_legacy_multiscale(existing, derived::Vector{Pair{Symbol,Any}}) + isempty(derived) && return existing + if isnothing(existing) + return derived + end + derived_inputs = Set(first(item) for item in derived) + merged = Pair{Any,Any}[] + for item in collect(existing) + mapped_input = first(item) + mapped_input = mapped_input isa PreviousTimeStep ? mapped_input.variable : mapped_input + mapped_input in derived_inputs && continue + push!(merged, item) + end + append!(merged, derived) + return merged +end diff --git a/src/time/multirate.jl b/src/time/multirate.jl index 1def05682..28aa12b10 100644 --- a/src/time/multirate.jl +++ b/src/time/multirate.jl @@ -222,6 +222,8 @@ mutable struct AggregateCache{T<:Real} <: OutputCache window_start::Float64 end +const OutputStream = Vector{Tuple{Float64,Any}} + """ ExportBuffer() @@ -244,6 +246,23 @@ end ExportBuffer(scale::Symbol, process::Symbol, var::Symbol) = ExportBuffer(scale, process, var, Int[], Int[], Any[]) +""" + OutputExportPlan + +Resolved online output export request used by the multi-rate runtime. +""" +struct OutputExportPlan{POL<:SchedulePolicy,C,MS} + name::Symbol + scale::Symbol + var::Symbol + process::Symbol + policy::POL + clock::C + model_spec::MS + source_dt::Float64 + source_sample_duration_seconds::Float64 +end + """ TemporalState(caches, last_run, streams, producer_horizons, export_plans, export_rows) TemporalState() @@ -261,9 +280,9 @@ interpolated policies. mutable struct TemporalState{ C<:AbstractDict{OutputKey,OutputCache}, L<:AbstractDict{ModelKey,Float64}, - S<:AbstractDict{OutputKey,Vector{Tuple{Float64,Any}}}, + S<:AbstractDict{OutputKey,OutputStream}, H<:AbstractDict{Tuple{Symbol,Symbol,Symbol},Float64}, - P<:AbstractVector, + P<:AbstractVector{<:OutputExportPlan}, R<:AbstractDict{Symbol,ExportBuffer} } caches::C @@ -277,8 +296,8 @@ end TemporalState() = TemporalState( Dict{OutputKey,OutputCache}(), Dict{ModelKey,Float64}(), - Dict{OutputKey,Vector{Tuple{Float64,Any}}}(), + Dict{OutputKey,OutputStream}(), Dict{Tuple{Symbol,Symbol,Symbol},Float64}(), - Any[], + OutputExportPlan[], Dict{Symbol,ExportBuffer}() ) diff --git a/src/time/runtime/bindings.jl b/src/time/runtime/bindings.jl index 25507bada..01ec5f484 100644 --- a/src/time/runtime/bindings.jl +++ b/src/time/runtime/bindings.jl @@ -7,6 +7,8 @@ Extract a symbol variable name from dependency graph variable descriptors function _symbol_from_dependency_var(x) if x isa Symbol return x + elseif x isa ProducerVariable + return x.input elseif x isa PreviousTimeStep return x.variable elseif x isa MappedVar @@ -17,10 +19,20 @@ function _symbol_from_dependency_var(x) end end +function _source_symbol_from_dependency_var(x) + if x isa ProducerVariable + return x.source + elseif x isa MappedVar + sv = source_variable(x) + return sv isa PreviousTimeStep ? sv.variable : sv + end + return _symbol_from_dependency_var(x) +end + """ _push_candidate_producer!(candidates, process_key, vars, input_var) -Append producer candidates `(process, input_var)` when `vars` contains +Append producer candidates `(process, source_var)` when `vars` contains `input_var`. """ function _push_candidate_producer!(candidates::Vector{Tuple{Symbol,Symbol}}, process_key, vars, input_var::Symbol) @@ -29,7 +41,8 @@ function _push_candidate_producer!(candidates::Vector{Tuple{Symbol,Symbol}}, pro s = _symbol_from_dependency_var(v) isnothing(s) && continue if s == input_var - push!(candidates, (process, input_var)) + source = _source_symbol_from_dependency_var(v) + source isa Symbol && push!(candidates, (process, source)) end end end @@ -78,8 +91,13 @@ function _parse_input_binding(binding) var = haskey(binding, :var) ? binding.var : nothing scale = if haskey(binding, :scale) sc = binding.scale - isnothing(sc) ? nothing : - (sc isa AbstractString ? _normalize_scale(sc; warn=true, context=:ModelSpec) : sc) + if isnothing(sc) + nothing + elseif sc isa Symbol + sc + else + error("Input binding scale must be a `Symbol`, got `$(typeof(sc))` for `$(repr(sc))`.") + end else nothing end diff --git a/src/time/runtime/environment_backends.jl b/src/time/runtime/environment_backends.jl new file mode 100644 index 000000000..9e777c90e --- /dev/null +++ b/src/time/runtime/environment_backends.jl @@ -0,0 +1,292 @@ +""" + AbstractEnvironmentBackend + +Backend protocol for meteorology and mutable microclimate providers. + +PlantSimEngine defines the protocol, not the spatial indexing strategy. External +packages can subtype this and implement `sample`, `scatter!`, `update_index!`, +`get_nsteps`, and `base_step_seconds`. +""" +abstract type AbstractEnvironmentBackend end + +""" + EnvironmentSupport(domain, scale, process, status) + +Minimal support descriptor passed to environment backends when a model samples +or scatters environmental variables. +""" +struct EnvironmentSupport{S} + domain::Symbol + scale::Symbol + process::Symbol + status::S +end + +""" + GlobalConstant(meteo) + +Environment backend that preserves the current PlantSimEngine behavior: every +model receives the same meteo object or meteo row at a given timestep. +""" +struct GlobalConstant{M} <: AbstractEnvironmentBackend + meteo::M +end + +environment_meteo(backend::GlobalConstant) = backend.meteo + +""" + environment_backend(meteo_or_backend) + +Return an environment backend. Plain meteorology is wrapped in +`GlobalConstant`; existing environment backends are returned unchanged. +""" +environment_backend(backend::AbstractEnvironmentBackend) = backend +environment_backend(meteo) = GlobalConstant(meteo) + +""" + base_step_seconds(backend) + +Return the duration of one simulation base step in seconds. +""" +function base_step_seconds(backend::AbstractEnvironmentBackend) + error( + "Environment backend `$(typeof(backend))` must implement ", + "`PlantSimEngine.base_step_seconds(backend)`." + ) +end + +function base_step_seconds(backend::GlobalConstant) + return _timeline_context(environment_meteo(backend)).base_step_seconds +end + +get_nsteps(backend::AbstractEnvironmentBackend) = error( + "Environment backend `$(typeof(backend))` must implement ", + "`PlantSimEngine.get_nsteps(backend)`." +) +get_nsteps(backend::GlobalConstant) = isnothing(environment_meteo(backend)) ? 1 : get_nsteps(environment_meteo(backend)) + +function _validate_meteo_duration(backend::AbstractEnvironmentBackend) + sec = base_step_seconds(backend) + sec isa Real && sec > 0 || error( + "Environment backend `$(typeof(backend))` returned invalid base step seconds `$(sec)`." + ) + return nothing +end + +_validate_meteo_duration(backend::GlobalConstant) = _validate_meteo_duration(environment_meteo(backend)) + +_timeline_context(backend::AbstractEnvironmentBackend) = TimelineContext(float(base_step_seconds(backend))) +_timeline_context(backend::GlobalConstant) = _timeline_context(environment_meteo(backend)) + +function _meteo_row_at_step(meteo, i::Int) + isnothing(meteo) && return nothing + get_nsteps(meteo) == 1 && return meteo + return first(Iterators.drop(Tables.rows(meteo), i - 1)) +end + +function _available_meteo_variables(meteo) + row = _first_meteo_row(meteo) + isnothing(row) && return nothing + return Set(Symbol.(propertynames(row))) +end + +""" + environment_variables(backend) + +Return a set of variable names that the backend can provide, or `nothing` when +the backend cannot enumerate them cheaply. +""" +environment_variables(::AbstractEnvironmentBackend) = nothing +function environment_variables(backend::GlobalConstant) + meteo = environment_meteo(backend) + isnothing(meteo) && return Set{Symbol}() + return _available_meteo_variables(meteo) +end + +function validate_meteo_inputs(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, backend::GlobalConstant) + return invoke( + validate_meteo_inputs, + Tuple{Dict{Symbol,Dict{Symbol,ModelSpec}},AbstractEnvironmentBackend}, + model_specs, + backend, + ) +end + +function validate_meteo_inputs(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, backend::AbstractEnvironmentBackend) + available = environment_variables(backend) + isnothing(available) && return nothing + + missing_rows = _collect_missing_meteo_rows(model_specs, var -> var in available) + return _error_missing_meteo_inputs( + missing_rows; + subject="Environment backend `$(typeof(backend))`", + noun="variables", + target="the backend" + ) +end + +""" + sample(backend, variable, support, time) + +Sample one environmental variable for a model support at a runtime time. +""" +function sample(backend::AbstractEnvironmentBackend, variable::Symbol, support::EnvironmentSupport, time) + error( + "Environment backend `$(typeof(backend))` must implement ", + "`PlantSimEngine.sample(backend, variable, support, time)`." + ) +end + +function sample(backend::GlobalConstant, variable::Symbol, support::EnvironmentSupport, time) + meteo = _meteo_row_at_step(environment_meteo(backend), Int(round(time))) + isnothing(meteo) && return nothing + hasproperty(meteo, variable) || error( + "GlobalConstant meteo does not provide variable `$(variable)` for `$(support.domain)/$(support.scale)/$(support.process)`." + ) + return getproperty(meteo, variable) +end + +""" + scatter!(backend, variable, support, value, time) + +Scatter one model-computed environmental value back to a mutable backend. +""" +function scatter!(backend::AbstractEnvironmentBackend, variable::Symbol, support::EnvironmentSupport, value, time) + error( + "Environment backend `$(typeof(backend))` does not implement ", + "`PlantSimEngine.scatter!(backend, variable, support, value, time)`." + ) +end + +scatter!(backend::GlobalConstant, variable::Symbol, support::EnvironmentSupport, value, time) = error( + "GlobalConstant is immutable and cannot receive environment output `$(variable)` from ", + "`$(support.domain)/$(support.scale)/$(support.process)`." +) + +""" + update_index!(backend, entities) + +Update the backend spatial/entity index after topology or geometry changes. +""" +update_index!(::AbstractEnvironmentBackend, entities) = nothing +update_index!(::GlobalConstant, entities) = nothing + +""" + sample_environment(backend, support, time, variables) + +Sample a meteo-like row for a model. `GlobalConstant` returns the original meteo +row; other backends return a `NamedTuple` assembled from `sample` calls. +""" +function sample_environment(backend::AbstractEnvironmentBackend, support::EnvironmentSupport, time, variables) + pairs = Pair{Symbol,Any}[] + for variable in variables + push!(pairs, variable => sample(backend, variable, support, time)) + end + return (; pairs...) +end + +function sample_environment(backend::GlobalConstant, support::EnvironmentSupport, time, variables) + return _meteo_row_at_step(environment_meteo(backend), Int(round(time))) +end + +function _environment_sampling_rules(model_spec::ModelSpec) + bindings = meteo_bindings(model_spec) + bindings = bindings isa NamedTuple ? bindings : NamedTuple() + rules = Pair{Symbol,Symbol}[] + for target in keys(meteo_inputs_(model_spec)) + source = target + if haskey(bindings, target) + rule = bindings[target] + rule isa NamedTuple && haskey(rule, :source) && (source = Symbol(rule.source)) + end + push!(rules, target => source) + end + return rules +end + +function sample_environment( + backend::AbstractEnvironmentBackend, + support::EnvironmentSupport, + time, + model_spec::ModelSpec +) + pairs = Pair{Symbol,Any}[] + for (target, source) in _environment_sampling_rules(model_spec) + push!(pairs, target => sample(backend, source, support, time)) + end + return (; pairs...) +end + +function sample_environment( + backend::GlobalConstant, + support::EnvironmentSupport, + time, + model_spec::ModelSpec +) + row = _meteo_row_at_step(environment_meteo(backend), Int(round(time))) + bindings = meteo_bindings(model_spec) + has_bindings = bindings isa NamedTuple && !isempty(keys(bindings)) + !has_bindings && return row + + pairs = Pair{Symbol,Any}[] + for (target, source) in _environment_sampling_rules(model_spec) + isnothing(row) && error( + "GlobalConstant meteo is `nothing`, but `$(support.domain)/$(support.scale)/$(support.process)` ", + "requires meteo variable `$(target)`." + ) + hasproperty(row, source) || error( + "GlobalConstant meteo does not provide source variable `$(source)` for model-facing variable ", + "`$(target)` in `$(support.domain)/$(support.scale)/$(support.process)`." + ) + push!(pairs, target => getproperty(row, source)) + end + if !isnothing(row) && hasproperty(row, :duration) + push!(pairs, :duration => getproperty(row, :duration)) + end + return (; pairs...) +end + +function _environment_output_value(status, variable::Symbol, support::EnvironmentSupport) + hasproperty(status, variable) && return getproperty(status, variable) + error( + "Model `$(support.domain)/$(support.scale)/$(support.process)` declares environment output ", + "`$(variable)` in `meteo_outputs_`, but its status does not contain `$(variable)`. ", + "Expose the computed value as a same-named `outputs_` variable or initialize it in the status." + ) +end + +""" + scatter_environment_outputs!(backend, support, time, model_spec, status) + +Scatter values declared by `meteo_outputs_(model)` from model status into a +mutable environment backend. +""" +function scatter_environment_outputs!( + backend::AbstractEnvironmentBackend, + support::EnvironmentSupport, + time, + model_spec::ModelSpec, + status +) + for variable in keys(meteo_outputs_(model_spec)) + value = _environment_output_value(status, variable, support) + scatter!(backend, variable, support, value, time) + end + return nothing +end + +""" + explain_environment(simulation) + +Return a compact description of the environment backend used by a domain +simulation. +""" +function explain_environment(simulation) + backend = simulation.environment + return ( + backend=typeof(backend), + variables=environment_variables(backend), + nsteps=get_nsteps(backend), + base_step_seconds=base_step_seconds(backend), + ) +end diff --git a/src/time/runtime/input_resolution.jl b/src/time/runtime/input_resolution.jl index 759a44d71..105dd8eb6 100644 --- a/src/time/runtime/input_resolution.jl +++ b/src/time/runtime/input_resolution.jl @@ -111,10 +111,10 @@ function _resolved_interpolated_value_for_source( return samples[1][2], true end -function _resolve_window_reducer(reducer) +function _normalize_time_reducer(reducer; context::AbstractString="reducer") if reducer isa DataType reducer <: PlantMeteo.AbstractTimeReducer || error( - "Unsupported reducer type `$(reducer)`. Use a PlantMeteo reducer type/instance or a callable." + "Unsupported $(context) type `$(reducer)`. Use a PlantMeteo reducer type/instance or a callable." ) return reducer() elseif reducer isa PlantMeteo.AbstractTimeReducer @@ -124,11 +124,13 @@ function _resolve_window_reducer(reducer) end error( - "Unsupported reducer value `$(reducer)` of type `$(typeof(reducer))`. ", + "Unsupported $(context) value `$(reducer)` of type `$(typeof(reducer))`. ", "Use a PlantMeteo reducer type/instance or a callable." ) end +_resolve_window_reducer(reducer) = _normalize_time_reducer(reducer; context="reducer") + function _window_reduce(vals::AbstractVector{<:Real}, durations::AbstractVector{<:Real}, policy::SchedulePolicy) reducer = policy isa Integrate ? policy.reducer : (policy isa Aggregate ? policy.reducer : PlantMeteo.SumReducer()) f = _resolve_window_reducer(reducer) @@ -198,105 +200,267 @@ function _prefer_local_status_fallback(st::Status, input_var::Symbol, source_var return true end -""" - _resolve_input_windowed(sim, node, st, input_var, source_scale, source_process, source_var, t_start, t_end, policy) +function _resolved_policy_value_for_source( + sim::GraphSimulation, + source_scope::ScopeId, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + source_node_id::Int, + t::Float64, + policy::HoldLast, + t_start::Float64, + source_sample_duration_seconds::Float64 +) + return _resolved_value_for_source(sim, source_scope, source_scale, source_process, source_var, source_node_id, t) +end -Resolve one consumer input from producer temporal streams using a windowed -policy (`Integrate` or `Aggregate`) and write it into `st`. -""" -function _resolve_input_windowed( +function _resolved_policy_value_for_source( + sim::GraphSimulation, + source_scope::ScopeId, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + source_node_id::Int, + t::Float64, + policy::Interpolate, + t_start::Float64, + source_sample_duration_seconds::Float64 +) + return _resolved_interpolated_value_for_source(sim, source_scope, source_scale, source_process, source_var, source_node_id, t, policy) +end + +function _resolved_policy_value_for_source( + sim::GraphSimulation, + source_scope::ScopeId, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + source_node_id::Int, + t::Float64, + policy::Union{Integrate,Aggregate}, + t_start::Float64, + source_sample_duration_seconds::Float64 +) + return _resolved_windowed_value_for_source( + sim, + source_scope, + source_scale, + source_process, + source_var, + source_node_id, + t_start, + t, + policy, + source_sample_duration_seconds + ) +end + +_missing_vector_source_value(policy::SchedulePolicy) = nothing +_missing_vector_source_value(policy::Union{Integrate,Aggregate}) = 0.0 +_missing_scalar_source_value(policy::SchedulePolicy) = nothing +_missing_scalar_source_value(policy::Union{Integrate,Aggregate}) = 0.0 + +function _source_scope_matches(sim::GraphSimulation, source_model_spec, source_scale::Symbol, source_process::Symbol, src_st::Status, consumer_scope::ScopeId) + source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) + return source_scope == consumer_scope, source_scope +end + +function _push_vector_source_value!( + vals::Vector{Any}, + sim::GraphSimulation, + src_st::Status, + consumer_scope::ScopeId, + source_model_spec, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + policy::SchedulePolicy, + t_start::Float64, + source_sample_duration_seconds::Float64 +) + scope_matches, source_scope = _source_scope_matches(sim, source_model_spec, source_scale, source_process, src_st, consumer_scope) + scope_matches || return nothing + + src_node_id = node_id(src_st.node) + v, ok = _resolved_policy_value_for_source( + sim, source_scope, source_scale, source_process, source_var, src_node_id, t, policy, t_start, source_sample_duration_seconds + ) + if ok + push!(vals, v) + elseif source_var in keys(src_st) + push!(vals, src_st[source_var]) + else + missing_value = _missing_vector_source_value(policy) + isnothing(missing_value) || push!(vals, missing_value) + end + return nothing +end + +function _assign_vector_source_values!( sim::GraphSimulation, - node::SoftDependencyNode, st::Status, consumer_scope::ScopeId, source_model_spec, - prefer_local_status::Bool, input_var::Symbol, + source_statuses, source_scale::Symbol, source_process::Symbol, source_var::Symbol, + t::Float64, + policy::SchedulePolicy, t_start::Float64, - t_end::Float64, + source_sample_duration_seconds::Float64 +) + vals = Any[] + for src_st in source_statuses + _push_vector_source_value!( + vals, + sim, + src_st, + consumer_scope, + source_model_spec, + source_scale, + source_process, + source_var, + t, + policy, + t_start, + source_sample_duration_seconds + ) + end + length(vals) > 0 && _assign_input_value!(st, input_var, vals) + return nothing +end + +function _resolve_ancestor_source_value( + sim::GraphSimulation, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + source_statuses, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, policy::SchedulePolicy, + t_start::Float64, source_sample_duration_seconds::Float64 ) - source_statuses = get(status(sim), source_scale, nothing) - isnothing(source_statuses) && return nothing + ancestor_node_id = _ancestor_node_id_for_scale(st.node, source_scale) + isnothing(ancestor_node_id) && return nothing, false + src_st = _status_for_node_id(source_statuses, ancestor_node_id) + isnothing(src_st) && return nothing, false - current_value = st[input_var] - if current_value isa AbstractVector - vals = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - v, ok = _resolved_windowed_value_for_source( - sim, source_scope, source_scale, source_process, source_var, src_node_id, t_start, t_end, policy, source_sample_duration_seconds - ) - if ok - push!(vals, v) - elseif policy isa Integrate || policy isa Aggregate - push!(vals, 0.0) - elseif source_var in keys(src_st) - push!(vals, src_st[source_var]) - end - end - length(vals) > 0 && _assign_input_value!(st, input_var, vals) - return nothing + scope_matches, source_scope = _source_scope_matches(sim, source_model_spec, source_scale, source_process, src_st, consumer_scope) + scope_matches || return nothing, false + + vv, found = _resolved_policy_value_for_source( + sim, source_scope, source_scale, source_process, source_var, ancestor_node_id, t, policy, t_start, source_sample_duration_seconds + ) + found && return vv, true + source_var in keys(src_st) && return src_st[source_var], true + return nothing, false +end + +function _collect_unique_source_candidates( + sim::GraphSimulation, + consumer_scope::ScopeId, + source_model_spec, + source_statuses, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + policy::SchedulePolicy, + t_start::Float64, + source_sample_duration_seconds::Float64 +) + candidates = Any[] + for src_st in source_statuses + scope_matches, source_scope = _source_scope_matches(sim, source_model_spec, source_scale, source_process, src_st, consumer_scope) + scope_matches || continue + src_node_id = node_id(src_st.node) + vv, found = _resolved_policy_value_for_source( + sim, source_scope, source_scale, source_process, source_var, src_node_id, t, policy, t_start, source_sample_duration_seconds + ) + found && push!(candidates, vv) end + return candidates +end +function _resolve_scalar_source_value!( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_statuses, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + policy::SchedulePolicy, + t_start::Float64, + source_sample_duration_seconds::Float64; + fallback_to_source_status::Bool +) _prefer_local_status_fallback(st, input_var, source_var, prefer_local_status) && return nothing consumer_node_id = node_id(st.node) - v, ok = _resolved_windowed_value_for_source( - sim, consumer_scope, source_scale, source_process, source_var, consumer_node_id, t_start, t_end, policy, source_sample_duration_seconds + v, ok = _resolved_policy_value_for_source( + sim, consumer_scope, source_scale, source_process, source_var, consumer_node_id, t, policy, t_start, source_sample_duration_seconds ) if ok _assign_input_value!(st, input_var, v) return nothing end - # Same-scale scalar fallback: prefer the value attached to the consumer node - # before scanning all source nodes (which can be ambiguous in dense scales). - if source_scale == node.scale - vv, found = _same_scale_status_value(source_statuses, consumer_node_id, source_var) - if found - _assign_input_value!(st, input_var, vv) - return nothing - end - else - ancestor_node_id = _ancestor_node_id_for_scale(st.node, source_scale) - if !isnothing(ancestor_node_id) - src_st = _status_for_node_id(source_statuses, ancestor_node_id) - if !isnothing(src_st) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - if source_scope == consumer_scope - vv, found = _resolved_windowed_value_for_source( - sim, source_scope, source_scale, source_process, source_var, ancestor_node_id, t_start, t_end, policy, source_sample_duration_seconds - ) - if found - _assign_input_value!(st, input_var, vv) - return nothing - elseif source_var in keys(src_st) - _assign_input_value!(st, input_var, src_st[source_var]) - return nothing - end - end + if fallback_to_source_status + if source_scale == node.scale + vv, found = _same_scale_status_value(source_statuses, consumer_node_id, source_var) + if found + _assign_input_value!(st, input_var, vv) + return nothing + end + else + vv, found = _resolve_ancestor_source_value( + sim, + st, + consumer_scope, + source_model_spec, + source_statuses, + source_scale, + source_process, + source_var, + t, + policy, + t_start, + source_sample_duration_seconds + ) + if found + _assign_input_value!(st, input_var, vv) + return nothing end end end - # Cross-scale scalar fallback: allow unique producer value at source scale. - candidates = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - vv, found = _resolved_windowed_value_for_source( - sim, source_scope, source_scale, source_process, source_var, src_node_id, t_start, t_end, policy, source_sample_duration_seconds - ) - found && push!(candidates, vv) - end + candidates = _collect_unique_source_candidates( + sim, + consumer_scope, + source_model_spec, + source_statuses, + source_scale, + source_process, + source_var, + t, + policy, + t_start, + source_sample_duration_seconds + ) if length(candidates) == 1 _assign_input_value!(st, input_var, only(candidates)) elseif length(candidates) > 1 @@ -304,20 +468,15 @@ function _resolve_input_windowed( "Ambiguous cross-scale source values for input `$(input_var)` in process `$(node.process)` at scale `$(node.scale)`. ", "Please provide `InputBindings(...)` with explicit `scale`/source disambiguation." ) - elseif policy isa Integrate || policy isa Aggregate - _assign_input_value!(st, input_var, 0.0) + else + missing_value = _missing_scalar_source_value(policy) + isnothing(missing_value) || _assign_input_value!(st, input_var, missing_value) end return nothing end -""" - _resolve_input_interpolate(sim, node, st, input_var, source_scale, source_process, source_var, t, policy) - -Resolve one consumer input from producer temporal streams using interpolation -policy and write it into `st`. -""" -function _resolve_input_interpolate( +function _resolve_input_from_policy!( sim::GraphSimulation, node::SoftDependencyNode, st::Status, @@ -329,63 +488,130 @@ function _resolve_input_interpolate( source_process::Symbol, source_var::Symbol, t::Float64, - policy::Interpolate + policy::SchedulePolicy, + t_start::Float64, + source_sample_duration_seconds::Float64; + fallback_to_source_status::Bool=true ) source_statuses = get(status(sim), source_scale, nothing) isnothing(source_statuses) && return nothing current_value = st[input_var] if current_value isa AbstractVector - vals = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - v, ok = _resolved_interpolated_value_for_source( - sim, source_scope, source_scale, source_process, source_var, src_node_id, t, policy - ) - if ok - push!(vals, v) - elseif source_var in keys(src_st) - push!(vals, src_st[source_var]) - end - end - length(vals) > 0 && _assign_input_value!(st, input_var, vals) - return nothing + return _assign_vector_source_values!( + sim, + st, + consumer_scope, + source_model_spec, + input_var, + source_statuses, + source_scale, + source_process, + source_var, + t, + policy, + t_start, + source_sample_duration_seconds + ) end - _prefer_local_status_fallback(st, input_var, source_var, prefer_local_status) && return nothing + return _resolve_scalar_source_value!( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_statuses, + source_scale, + source_process, + source_var, + t, + policy, + t_start, + source_sample_duration_seconds; + fallback_to_source_status=fallback_to_source_status + ) +end - consumer_node_id = node_id(st.node) - v, ok = _resolved_interpolated_value_for_source( - sim, consumer_scope, source_scale, source_process, source_var, consumer_node_id, t, policy +""" + _resolve_input_windowed(sim, node, st, input_var, source_scale, source_process, source_var, t_start, t_end, policy) + +Resolve one consumer input from producer temporal streams using a windowed +policy (`Integrate` or `Aggregate`) and write it into `st`. +""" +function _resolve_input_windowed( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t_start::Float64, + t_end::Float64, + policy::SchedulePolicy, + source_sample_duration_seconds::Float64 +) + return _resolve_input_from_policy!( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t_end, + policy, + t_start, + source_sample_duration_seconds ) - if ok - _assign_input_value!(st, input_var, v) - return nothing - end +end - # Cross-scale scalar fallback: allow unique producer value at source scale. - candidates = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - vv, found = _resolved_interpolated_value_for_source( - sim, source_scope, source_scale, source_process, source_var, src_node_id, t, policy - ) - found && push!(candidates, vv) - end - if length(candidates) == 1 - _assign_input_value!(st, input_var, only(candidates)) - elseif length(candidates) > 1 - error( - "Ambiguous cross-scale source values for input `$(input_var)` in process `$(node.process)` at scale `$(node.scale)`. ", - "Please provide `InputBindings(...)` with explicit `scale`/source disambiguation." - ) - end +""" + _resolve_input_interpolate(sim, node, st, input_var, source_scale, source_process, source_var, t, policy) - return nothing +Resolve one consumer input from producer temporal streams using interpolation +policy and write it into `st`. +""" +function _resolve_input_interpolate( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + policy::Interpolate +) + return _resolve_input_from_policy!( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t, + policy, + t, + 0.0; + fallback_to_source_status=false + ) end """ @@ -407,85 +633,141 @@ function _resolve_input_holdlast( source_var::Symbol, t::Float64 ) - source_statuses = get(status(sim), source_scale, nothing) - isnothing(source_statuses) && return nothing - - current_value = st[input_var] - if current_value isa AbstractVector - vals = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - v, ok = _resolved_value_for_source(sim, source_scope, source_scale, source_process, source_var, src_node_id, t) - if ok - push!(vals, v) - else - if source_var in keys(src_st) - push!(vals, src_st[source_var]) - end - end - end - length(vals) > 0 && _assign_input_value!(st, input_var, vals) - return nothing - end - - _prefer_local_status_fallback(st, input_var, source_var, prefer_local_status) && return nothing + return _resolve_input_from_policy!( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t, + HoldLast(), + t, + 0.0 + ) +end - consumer_node_id = node_id(st.node) - v, ok = _resolved_value_for_source(sim, consumer_scope, source_scale, source_process, source_var, consumer_node_id, t) - if ok - _assign_input_value!(st, input_var, v) - return nothing - end +function _resolve_input_for_policy!( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + t_start::Float64, + policy::HoldLast, + source_sample_duration_seconds::Float64 +) + return _resolve_input_holdlast( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t + ) +end - # Same-scale scalar fallback: prefer the value attached to the consumer node - # before scanning all source nodes (which can be ambiguous in dense scales). - if source_scale == node.scale - vv, found = _same_scale_status_value(source_statuses, consumer_node_id, source_var) - if found - _assign_input_value!(st, input_var, vv) - return nothing - end - else - ancestor_node_id = _ancestor_node_id_for_scale(st.node, source_scale) - if !isnothing(ancestor_node_id) - src_st = _status_for_node_id(source_statuses, ancestor_node_id) - if !isnothing(src_st) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - if source_scope == consumer_scope - vv, found = _resolved_value_for_source(sim, source_scope, source_scale, source_process, source_var, ancestor_node_id, t) - if found - _assign_input_value!(st, input_var, vv) - return nothing - elseif source_var in keys(src_st) - _assign_input_value!(st, input_var, src_st[source_var]) - return nothing - end - end - end - end - end +function _resolve_input_for_policy!( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + t_start::Float64, + policy::Interpolate, + source_sample_duration_seconds::Float64 +) + return _resolve_input_interpolate( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t, + policy + ) +end - # Cross-scale scalar fallback: allow unique producer value at source scale. - candidates = Any[] - for src_st in source_statuses - src_node_id = node_id(src_st.node) - source_scope = _scope_for_status(sim, source_model_spec, source_scale, source_process, src_st.node) - source_scope == consumer_scope || continue - vv, found = _resolved_value_for_source(sim, source_scope, source_scale, source_process, source_var, src_node_id, t) - found && push!(candidates, vv) - end - if length(candidates) == 1 - _assign_input_value!(st, input_var, only(candidates)) - elseif length(candidates) > 1 - error( - "Ambiguous cross-scale source values for input `$(input_var)` in process `$(node.process)` at scale `$(node.scale)`. ", - "Please provide `InputBindings(...)` with explicit `scale`/source disambiguation." - ) - end +function _resolve_input_for_policy!( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + t_start::Float64, + policy::Union{Integrate,Aggregate}, + source_sample_duration_seconds::Float64 +) + return _resolve_input_windowed( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t_start, + t, + policy, + source_sample_duration_seconds + ) +end - return nothing +function _resolve_input_for_policy!( + sim::GraphSimulation, + node::SoftDependencyNode, + st::Status, + consumer_scope::ScopeId, + source_model_spec, + prefer_local_status::Bool, + input_var::Symbol, + source_scale::Symbol, + source_process::Symbol, + source_var::Symbol, + t::Float64, + t_start::Float64, + policy::SchedulePolicy, + source_sample_duration_seconds::Float64 +) + error( + "Unsupported input temporal policy `$(typeof(policy))` for input `$(input_var)` ", + "in process `$(node.process)` at scale `$(node.scale)`." + ) end """ @@ -541,28 +823,22 @@ function resolve_inputs_from_temporal_state!(sim::GraphSimulation, node::SoftDep policy = _policy_for_output(model_(source_model_spec), source_var) end - if policy isa HoldLast - _resolve_input_holdlast(sim, node, st, consumer_scope, source_model_spec, prefer_local_status, input_var, source_scale, source_process, source_var, t) - elseif policy isa Interpolate - _resolve_input_interpolate(sim, node, st, consumer_scope, source_model_spec, prefer_local_status, input_var, source_scale, source_process, source_var, t, policy) - elseif policy isa Integrate || policy isa Aggregate - _resolve_input_windowed( - sim, - node, - st, - consumer_scope, - source_model_spec, - prefer_local_status, - input_var, - source_scale, - source_process, - source_var, - t_start, - t, - policy, - source_sample_duration_seconds - ) - end + _resolve_input_for_policy!( + sim, + node, + st, + consumer_scope, + source_model_spec, + prefer_local_status, + input_var, + source_scale, + source_process, + source_var, + t, + t_start, + policy, + source_sample_duration_seconds + ) end return nothing diff --git a/src/time/runtime/meteo_sampling.jl b/src/time/runtime/meteo_sampling.jl index 8a80f62bf..c46d08e3d 100644 --- a/src/time/runtime/meteo_sampling.jl +++ b/src/time/runtime/meteo_sampling.jl @@ -11,22 +11,7 @@ function _prepare_meteo_sampler(meteo) end function _runtime_meteo_window(window) - if isnothing(window) - return nothing - elseif window isa PlantMeteo.AbstractSamplingWindow - return window - elseif window isa DataType - window <: PlantMeteo.AbstractSamplingWindow || error( - "Unsupported MeteoWindow type `$(window)`. ", - "Use a PlantMeteo sampling-window type/instance." - ) - return window() - end - - error( - "Unsupported MeteoWindow value `$(window)` of type `$(typeof(window))`. ", - "Use a PlantMeteo sampling-window type/instance." - ) + return _normalize_meteo_window(window) end function _meteo_sampling_window(clock::ClockSpec, model_spec) @@ -42,24 +27,7 @@ function _meteo_sampling_window(clock::ClockSpec, model_spec) return window end -function _normalize_meteo_reducer(reducer) - if reducer isa DataType - reducer <: PlantMeteo.AbstractTimeReducer || error( - "Unsupported meteo reducer type `$(reducer)`. ", - "Use a PlantMeteo reducer type/instance or a callable." - ) - return reducer() - elseif reducer isa PlantMeteo.AbstractTimeReducer - return reducer - elseif reducer isa Function - return reducer - end - - error( - "Unsupported meteo reducer value `$(reducer)` of type `$(typeof(reducer))`. ", - "Use a PlantMeteo reducer type/instance or a callable." - ) -end +_normalize_meteo_reducer(reducer) = _normalize_time_reducer(reducer; context="meteo reducer") function _normalize_meteo_binding_rule(target::Symbol, rule) if rule isa NamedTuple @@ -76,6 +44,144 @@ function _normalize_meteo_binding_rule(target::Symbol, rule) ) end +function _raw_meteo_requirements_for_spec(model_spec) + required = Set{Symbol}(keys(meteo_inputs_(model_spec))) + isempty(required) && return required + + raw_required = Set{Symbol}() + bindings = meteo_bindings(model_spec) + bindings = bindings isa NamedTuple ? bindings : NamedTuple() + for var in required + if haskey(bindings, var) + rule = bindings[var] + if rule isa NamedTuple && haskey(rule, :source) + push!(raw_required, Symbol(rule.source)) + else + push!(raw_required, var) + end + else + push!(raw_required, var) + end + end + + return raw_required +end + +function _first_meteo_row(meteo) + isnothing(meteo) && return nothing + is_table = try + DataFormat(meteo) == TableAlike() + catch + false + end + if is_table + rows = Tables.rows(meteo) + state = iterate(rows) + isnothing(state) && return nothing + return state[1] + end + return meteo +end + +_meteo_has_field(row, var::Symbol) = hasproperty(row, var) +_meteo_has_field(row::NamedTuple, var::Symbol) = haskey(row, var) + +function _collect_missing_meteo_rows(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, has_meteo_variable) + missing_rows = NamedTuple[] + for (scale, specs_at_scale) in model_specs + for (process, spec) in specs_at_scale + required = _raw_meteo_requirements_for_spec(spec) + missing = Symbol[var for var in required if !has_meteo_variable(var)] + isempty(missing) && continue + push!(missing_rows, (scale=scale, process=process, missing=Tuple(missing))) + end + end + return missing_rows +end + +function _format_missing_meteo_rows(missing_rows) + return join( + [ + string(row.scale, "/", row.process, " missing ", row.missing) + for row in missing_rows + ], + "; " + ) +end + +function _error_missing_meteo_inputs(missing_rows; subject::AbstractString, noun::AbstractString, target::AbstractString) + isempty(missing_rows) && return nothing + + error( + subject, + " is missing ", + noun, + " required by model `meteo_inputs_`: ", + _format_missing_meteo_rows(missing_rows), + ". Add the ", + noun, + " to ", + target, + ", declare a `MeteoBindings(source=...)` remapping, ", + "or remove the unused meteo input from the model trait." + ) +end + +""" + validate_meteo_inputs(model_specs, meteo) + +Validate declared `meteo_inputs_` against the available meteorological fields. + +The check is intentionally field-based and independent from units/backends. When +`MeteoBindings` remap a declared model input from another source variable, the +source variable is checked on the raw meteo object. +""" +function validate_meteo_inputs(model_specs::Dict{Symbol,Dict{Symbol,ModelSpec}}, meteo) + row = _first_meteo_row(meteo) + isnothing(row) && return nothing + + missing_rows = _collect_missing_meteo_rows(model_specs, var -> _meteo_has_field(row, var)) + return _error_missing_meteo_inputs( + missing_rows; + subject="Meteorology", + noun="fields", + target="meteo" + ) +end + +function validate_meteo_inputs(model_specs::AbstractDict{Symbol,<:AbstractDict}, meteo) + normalized_specs = Dict{Symbol,Dict{Symbol,ModelSpec}}() + for (scale, specs_at_scale) in pairs(model_specs) + normalized_specs[scale] = Dict{Symbol,ModelSpec}( + Symbol(process) => as_model_spec(spec) for (process, spec) in pairs(specs_at_scale) + ) + end + return validate_meteo_inputs(normalized_specs, meteo) +end + +function validate_meteo_inputs(models::NamedTuple, meteo) + specs = Dict( + :Default => Dict{Symbol,ModelSpec}( + process(model) => as_model_spec(model) for model in values(models) + ) + ) + return validate_meteo_inputs(specs, meteo) +end + +function validate_meteo_inputs(mapping::ModelMapping, meteo) + specs = Dict{Symbol,Dict{Symbol,ModelSpec}}( + scale => parse_model_specs(declarations) for (scale, declarations) in pairs(mapping) + ) + return validate_meteo_inputs(specs, meteo) +end + +function validate_meteo_inputs(mapping::AbstractDict, meteo) + specs = Dict{Symbol,Dict{Symbol,ModelSpec}}( + Symbol(scale) => parse_model_specs(declarations) for (scale, declarations) in pairs(mapping) + ) + return validate_meteo_inputs(specs, meteo) +end + function _meteo_transforms_for_model(model_spec) bindings = meteo_bindings(model_spec) isnothing(bindings) && return nothing diff --git a/src/time/runtime/output_export.jl b/src/time/runtime/output_export.jl index 1565a83db..53e603ad1 100644 --- a/src/time/runtime/output_export.jl +++ b/src/time/runtime/output_export.jl @@ -58,24 +58,6 @@ function OutputRequest( return OutputRequest(scale, var, name, proc, policy, clock) end -function OutputRequest( - scale::AbstractString, - var::Symbol; - name::Symbol=var, - process=nothing, - policy::SchedulePolicy=HoldLast(), - clock=nothing -) - return OutputRequest( - _normalize_scale(scale; warn=true, context=:OutputRequest), - var; - name=name, - process=process, - policy=policy, - clock=clock - ) -end - function _export_clock(request::OutputRequest, timeline::TimelineContext) isnothing(request.clock) && return ClockSpec(1.0, 0.0) c = _clock_from_spec_timestep(request.clock, timeline) @@ -89,8 +71,8 @@ function _canonical_source_process(sim::GraphSimulation, scale::Symbol, var::Sym haskey(get_models(sim), scale) || error("Unknown scale `$(scale)` in output export request.") models_at_scale = get_models(sim)[scale] specs_at_scale = get_model_specs(sim)[scale] - ignored_same_rate_hard_children = _same_rate_hard_dependency_children(get_model_specs(sim), dep(sim)) - ignored_at_scale = get(ignored_same_rate_hard_children, scale, Set{Symbol}()) + ignored_hard_children = _hard_dependency_children(dep(sim)) + ignored_at_scale = get(ignored_hard_children, scale, Set{Symbol}()) publishers = Symbol[] for (process, model) in pairs(models_at_scale) @@ -134,7 +116,7 @@ Resolve and register online export requests for the current run. function prepare_output_requests!(sim::GraphSimulation, requests, timeline::TimelineContext) reqs = _normalize_output_requests(requests) - plans = Any[] + plans = OutputExportPlan[] rows = Dict{Symbol,ExportBuffer}() for req in reqs @@ -149,17 +131,20 @@ function prepare_output_requests!(sim::GraphSimulation, requests, timeline::Time "Duplicate output request name `$(req.name)`. Request names must be unique." ) - push!(plans, ( - name=req.name, - scale=scale, - var=req.var, - process=process, - policy=req.policy, - clock=clock, - model_spec=model_spec, - source_dt=float(source_clock.dt), - source_sample_duration_seconds=float(source_clock.dt) * timeline.base_step_seconds, - )) + push!( + plans, + OutputExportPlan( + req.name, + scale, + req.var, + process, + req.policy, + clock, + model_spec, + float(source_clock.dt), + float(source_clock.dt) * timeline.base_step_seconds, + ), + ) rows[req.name] = ExportBuffer(scale, process, req.var) end @@ -169,13 +154,7 @@ function prepare_output_requests!(sim::GraphSimulation, requests, timeline::Time end function _required_horizon_for_export_policy(policy::SchedulePolicy, clock::ClockSpec, source_dt::Float64) - if policy isa Union{Integrate,Aggregate} - return max(1.0, float(clock.dt)) - elseif policy isa Interpolate - return max(2.0, source_dt + 1.0) - end - # HoldLast export is served from caches and does not require streams. - return 0.0 + return _required_horizon_for_policy(policy, float(clock.dt), source_dt) end """ diff --git a/src/time/runtime/publishers.jl b/src/time/runtime/publishers.jl index ff314ece2..ce044dc5c 100644 --- a/src/time/runtime/publishers.jl +++ b/src/time/runtime/publishers.jl @@ -32,10 +32,10 @@ Ensure that each `(scale, variable)` has at most one canonical publisher. Throws when multiple producers publish the same canonical output. """ function validate_canonical_publishers(sim::GraphSimulation) - ignored_same_rate_hard_children = _same_rate_hard_dependency_children(get_model_specs(sim), dep(sim)) + ignored_hard_children = _hard_dependency_children(dep(sim)) for (scale, models_at_scale) in get_models(sim) specs_at_scale = get_model_specs(sim)[scale] - ignored_at_scale = get(ignored_same_rate_hard_children, scale, Set{Symbol}()) + ignored_at_scale = get(ignored_hard_children, scale, Set{Symbol}()) publishers = Dict{Symbol,Vector{Symbol}}() for (process, model) in pairs(models_at_scale) process in ignored_at_scale && continue @@ -52,10 +52,21 @@ function validate_canonical_publishers(sim::GraphSimulation) for (var, procs) in publishers if length(procs) > 1 + updater_flags = Dict( + process => (var in _update_variables_for_spec(get(specs_at_scale, process, as_model_spec(models_at_scale[process])))) + for process in procs + ) + primary_procs = [process for process in procs if !updater_flags[process]] + if length(primary_procs) == 1 + update_procs = [process for process in procs if updater_flags[process]] + primary = only(primary_procs) + all(process -> primary in _update_after_for_var(specs_at_scale[process], var), update_procs) && continue + end error( "Ambiguous canonical publishers for variable `$(var)` at scale `$(scale)`: ", join(procs, ", "), - ". Declare `OutputRouting(; $(var)=:stream_only)` for non-canonical producers." + ". Declare `OutputRouting(; $(var)=:stream_only)` for non-canonical producers, ", + "or `Updates(:$(var); after=:primary_process)` for intentional state updates." ) end end @@ -96,9 +107,9 @@ function _consumer_horizon_requirements(sim::GraphSimulation, timeline::Timeline source_process = parsed.process source_var = isnothing(parsed.var) ? input_var : parsed.var source_scale = isnothing(parsed.scale) ? _source_scale_for_process(node, source_process) : parsed.scale - source_scale = source_scale isa AbstractString ? - _normalize_scale(source_scale; warn=true, context=:ModelSpec) : - source_scale + source_scale isa Symbol || error( + "Source scale for input `$(input_var)` in process `$(node.value)` must be a `Symbol`, got `$(typeof(source_scale))`." + ) source_model_spec = _model_spec_for_process(sim, source_scale, source_process) source_model = get_models(sim)[source_scale][source_process] source_clock = _model_clock(source_model_spec, source_model, timeline) @@ -130,7 +141,7 @@ function configure_temporal_buffers!(sim::GraphSimulation, timeline::TimelineCon return nothing end -function _trim_stream!(samples::Vector{Tuple{Float64,Any}}, t::Float64, horizon::Float64) +function _trim_stream!(samples::OutputStream, t::Float64, horizon::Float64) horizon <= 0.0 && (empty!(samples); return nothing) t_min = t - horizon + 1.0 - 1e-8 first_keep = findfirst(s -> s[1] >= t_min, samples) @@ -161,7 +172,7 @@ function update_temporal_state_outputs!(sim::GraphSimulation, node::SoftDependen producer_key = _producer_signature(node.scale, node.process, out_var) if haskey(sim.temporal_state.producer_horizons, producer_key) - samples = get!(sim.temporal_state.streams, key, Vector{Tuple{Float64,Any}}()) + samples = get!(sim.temporal_state.streams, key, OutputStream()) push!(samples, (t, val)) _trim_stream!(samples, t, sim.temporal_state.producer_horizons[producer_key]) end diff --git a/src/time/runtime/scopes.jl b/src/time/runtime/scopes.jl index 2c5c00e4b..7a46d490b 100644 --- a/src/time/runtime/scopes.jl +++ b/src/time/runtime/scopes.jl @@ -26,7 +26,6 @@ function _find_ancestor_by_symbol(node, target::Symbol) end return nothing end -_find_ancestor_by_symbol(node, target::AbstractString) = _find_ancestor_by_symbol(node, Symbol(target)) function _scope_from_builtin(selector::Symbol, node, scale::Symbol, process::Symbol) if selector == :global @@ -60,12 +59,10 @@ function _scope_from_selector_result(result, node, scale::Symbol, process::Symbo return result elseif result isa Symbol return _scope_from_builtin(result, node, scale, process) - elseif result isa AbstractString - return _scope_from_builtin(Symbol(result), node, scale, process) end error( - "Scope selector for process `$(process)` at scale `$(scale)` must return `ScopeId`, `Symbol`, or `String`, ", + "Scope selector for process `$(process)` at scale `$(scale)` must return `ScopeId` or `Symbol`, ", "got `$(typeof(result))`." ) end @@ -75,8 +72,6 @@ function _scope_from_selector(selector, node, scale::Symbol, process::Symbol) return selector elseif selector isa Symbol return _scope_from_builtin(selector, node, scale, process) - elseif selector isa AbstractString - return _scope_from_builtin(Symbol(selector), node, scale, process) elseif selector isa Function result = if applicable(selector, node, scale, process) selector(node, scale, process) diff --git a/src/traits/table_traits.jl b/src/traits/table_traits.jl index e4eab2f83..2ba21e17a 100644 --- a/src/traits/table_traits.jl +++ b/src/traits/table_traits.jl @@ -17,7 +17,7 @@ how to iterate over the data. The following data formats are supported: The default implementation returns `TableAlike` for `AbstractDataFrame`, `TimeStepTable`, `AbstractVector` and `Dict`, `TreeAlike` for `GraphSimulation`, -`SingletonAlike` for `Status`, `ModelList`, `NamedTuple` and `TimeStepRow`. +`SingletonAlike` for `Status`, `SingleScaleModelSet`, `NamedTuple` and `TimeStepRow`. The default implementation for `Any` throws an error. Users that want to use another input should define this trait for the new data format, e.g.: @@ -51,13 +51,13 @@ DataFormat(::Type{<:DataFrames.AbstractDataFrame}) = TableAlike() DataFormat(::Type{<:PlantMeteo.TimeStepTable}) = TableAlike() DataFormat(::Type{<:PlantMeteo.TimeStepRows}) = TableAlike() -# Giving a ModelList as a vector or a dict of objects: +# Giving a SingleScaleModelSet as a vector or a dict of objects: DataFormat(::Type{<:AbstractVector}) = TableAlike() DataFormat(::Type{<:Dict}) = TableAlike() DataFormat(::Type{<:NamedTuple}) = SingletonAlike() DataFormat(::Type{<:Status}) = SingletonAlike() -DataFormat(::Type{<:ModelList{Mo,S} where {Mo,S}}) = SingletonAlike() +DataFormat(::Type{<:SingleScaleModelSet{Mo,S} where {Mo,S}}) = SingletonAlike() DataFormat(::Type{<:ModelMapping{SingleScale}}) = SingletonAlike() DataFormat(::Type{<:GraphSimulation}) = TreeAlike() diff --git a/test/helper-functions.jl b/test/helper-functions.jl index 0e1524b66..588857b56 100644 --- a/test/helper-functions.jl +++ b/test/helper-functions.jl @@ -44,6 +44,19 @@ function compare_outputs_modellist_mapping(filtered_outputs_modellist, graphsim) return modellist_sorted == mapping_sorted end +function run_graphsim_for_comparison(mtg, mapping, nsteps, outputs_mapping, meteo) + graphsim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=outputs_mapping) + run!( + graphsim, + meteo, + PlantMeteo.Constants(), + nothing; + check=true, + executor=SequentialEx(), + ) + return graphsim +end + # Helper used to compare a single-scale `ModelMapping` run with its generated # multiscale equivalent. function check_multiscale_simulation_is_equivalent_begin(mapping::ModelMapping, meteo) @@ -55,16 +68,7 @@ function check_multiscale_simulation_is_equivalent_begin(mapping::ModelMapping, end function check_multiscale_simulation_is_equivalent_end(modellist_outputs, mtg, mapping, out, meteo) - graph_sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=PlantSimEngine.get_nsteps(meteo), check=true, outputs=out) - - sim = run!(graph_sim, - meteo, - PlantMeteo.Constants(), - nothing; - check=true, - executor=SequentialEx() - ) - + graph_sim = run_graphsim_for_comparison(mtg, mapping, PlantSimEngine.get_nsteps(meteo), out, meteo) return compare_outputs_modellist_mapping(modellist_outputs, graph_sim) end @@ -394,15 +398,7 @@ function test_filtered_output_begin(m::ModelMapping, status_tuple, requested_out end function test_filtered_output(mtg, mapping, nsteps, outputs_mapping, meteo, filtered_outputs_modellist) - graphsim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=outputs_mapping) - - sim2 = run!(graphsim, - meteo, - PlantMeteo.Constants(), - nothing; - check=true, - executor=SequentialEx() - ) + graphsim = run_graphsim_for_comparison(mtg, mapping, nsteps, outputs_mapping, meteo) return compare_outputs_modellist_mapping(filtered_outputs_modellist, graphsim) end diff --git a/test/runtests.jl b/test/runtests.jl index 99cc88cdb..454f6d120 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -30,6 +30,10 @@ include("helper-functions.jl") include("test-multirate-scaffolding.jl") end + @testset "Unified scene/object API" begin + include("test-unified-scene-object-api.jl") + end + @testset "Multi-rate runtime" begin include("test-multirate-runtime.jl") end @@ -38,6 +42,26 @@ include("helper-functions.jl") include("test-multirate-output-export.jl") end + @testset "ModelSpec Updates" begin + include("test-updates.jl") + end + + @testset "Meteo traits" begin + include("test-meteo-traits.jl") + end + + @testset "Environment backends" begin + include("test-environment-backends.jl") + end + + @testset "Domain simulation" begin + include("test-domain-simulation.jl") + end + + @testset "MAESPA-style domain example" begin + include("test-maespa-domain-example.jl") + end + @testset "MultiScaleModel" begin include("test-MultiScaleModel.jl") end diff --git a/test/test-MultiScaleModel.jl b/test/test-MultiScaleModel.jl index 5096fa886..a8e02cbaa 100644 --- a/test/test-MultiScaleModel.jl +++ b/test/test-MultiScaleModel.jl @@ -11,9 +11,9 @@ @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => (:Plant => :plant_surfaces)) == (PreviousTimeStep(:plant_surfaces, :unknown) => :Plant => :plant_surfaces) # Case 6 @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => :Plant => :surface) == (PreviousTimeStep(:plant_surfaces, :unknown) => :Plant => :surface) # Case 6 @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => [:Plant => :surface, :Leaf => :surface]) == (PreviousTimeStep(:plant_surfaces, :unknown) => [:Plant => :surface, :Leaf => :surface]) # Case 6 - @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces)) == (PreviousTimeStep(:plant_surfaces, :unknown) => Symbol("") => :plant_surfaces) # Case 7 - @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => (Symbol("") => :surface)) == (PreviousTimeStep(:plant_surfaces, :unknown) => Symbol("") => :surface) - @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => (Symbol("") => :surface), :test) == (PreviousTimeStep(:plant_surfaces, :test) => Symbol("") => :surface) + @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces)) == (PreviousTimeStep(:plant_surfaces, :unknown) => SameScale() => :plant_surfaces) # Case 7 + @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => (SameScale() => :surface)) == (PreviousTimeStep(:plant_surfaces, :unknown) => SameScale() => :surface) + @test PlantSimEngine._get_var(PreviousTimeStep(:plant_surfaces) => (SameScale() => :surface), :test) == (PreviousTimeStep(:plant_surfaces, :test) => SameScale() => :surface) end; @testset "MultiScaleModel: case 1" begin diff --git a/test/test-domain-simulation.jl b/test/test-domain-simulation.jl new file mode 100644 index 000000000..fc25ff57f --- /dev/null +++ b/test/test-domain-simulation.jl @@ -0,0 +1,1521 @@ +using Dates + +PlantSimEngine.@process "domain_absorbed_radiation" verbose = false +PlantSimEngine.@process "domain_plant_transpiration" verbose = false +PlantSimEngine.@process "domain_soil_water" verbose = false +PlantSimEngine.@process "domain_soil_evaporation" verbose = false +PlantSimEngine.@process "domain_scene_evapotranspiration" verbose = false +PlantSimEngine.@process "domain_scene_plant_evapotranspiration" verbose = false +PlantSimEngine.@process "domain_hard_leaf_conductance" verbose = false +PlantSimEngine.@process "domain_hard_leaf_energy" verbose = false +PlantSimEngine.@process "domain_scene_conductance_sum" verbose = false +PlantSimEngine.@process "domain_hard_target_signal" verbose = false +PlantSimEngine.@process "domain_scene_hard_target_sum" verbose = false +PlantSimEngine.@process "domain_scene_calls_target_sum" verbose = false +PlantSimEngine.@process "domain_hard_target_leaf_counter" verbose = false +PlantSimEngine.@process "domain_scene_hard_target_leaf_sum" verbose = false +PlantSimEngine.@process "domain_scene_routed_vector" verbose = false +PlantSimEngine.@process "domain_scene_routed_aggregate" verbose = false +PlantSimEngine.@process "domain_mtg_leaf_flux" verbose = false +PlantSimEngine.@process "domain_scene_dependency_flux_sum" verbose = false +PlantSimEngine.@process "domain_mtg_leaf_soil_flux" verbose = false +PlantSimEngine.@process "domain_growth_leaf_emergence" verbose = false +PlantSimEngine.@process "domain_growth_leaf_flux" verbose = false +PlantSimEngine.@process "domain_growth_integrated_flux" verbose = false +PlantSimEngine.@process "domain_scene_growth_flux_sum" verbose = false +PlantSimEngine.@process "domain_update_allocation" verbose = false +PlantSimEngine.@process "domain_update_pruning" verbose = false +PlantSimEngine.@process "domain_update_observer" verbose = false +PlantSimEngine.@process "domain_removal_leaf_flux" verbose = false +PlantSimEngine.@process "domain_removal_pruning" verbose = false +PlantSimEngine.@process "domain_churn_leaf_flux" verbose = false +PlantSimEngine.@process "domain_churn_leaf_controller" verbose = false +PlantSimEngine.@process "domain_subtree_internode_flux" verbose = false +PlantSimEngine.@process "domain_subtree_leaf_flux" verbose = false +PlantSimEngine.@process "domain_subtree_pruning" verbose = false +PlantSimEngine.@process "domain_reparent_leaf_controller" verbose = false + +struct DomainAbsorbedRadiationModel <: AbstractDomain_Absorbed_RadiationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DomainAbsorbedRadiationModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainAbsorbedRadiationModel) = (absorbed_radiation=0.0,) +PlantSimEngine.meteo_inputs_(::DomainAbsorbedRadiationModel) = (Ri_PAR_f=0.0,) + +function PlantSimEngine.run!(model::DomainAbsorbedRadiationModel, models, status, meteo, constants=nothing, extra=nothing) + status.absorbed_radiation = model.coefficient * meteo.Ri_PAR_f + return nothing +end + +struct DomainPlantTranspirationModel <: AbstractDomain_Plant_TranspirationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DomainPlantTranspirationModel) = (absorbed_radiation=0.0,) +PlantSimEngine.outputs_(::DomainPlantTranspirationModel) = (transpiration=0.0,) + +function PlantSimEngine.run!(model::DomainPlantTranspirationModel, models, status, meteo, constants=nothing, extra=nothing) + status.transpiration = model.coefficient * status.absorbed_radiation + return nothing +end + +struct DomainSoilWaterModel <: AbstractDomain_Soil_WaterModel + baseline::Float64 +end + +PlantSimEngine.inputs_(::DomainSoilWaterModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSoilWaterModel) = (soil_water_content=0.0,) +PlantSimEngine.meteo_inputs_(::DomainSoilWaterModel) = (T=0.0,) + +function PlantSimEngine.run!(model::DomainSoilWaterModel, models, status, meteo, constants=nothing, extra=nothing) + status.soil_water_content = model.baseline - 0.001 * meteo.T + return nothing +end + +struct DomainSoilEvaporationModel <: AbstractDomain_Soil_EvaporationModel + coefficient::Float64 +end + +PlantSimEngine.inputs_(::DomainSoilEvaporationModel) = (soil_water_content=0.0,) +PlantSimEngine.outputs_(::DomainSoilEvaporationModel) = (evaporation=0.0,) +PlantSimEngine.meteo_inputs_(::DomainSoilEvaporationModel) = (T=0.0,) + +function PlantSimEngine.run!(model::DomainSoilEvaporationModel, models, status, meteo, constants=nothing, extra=nothing) + status.evaporation = model.coefficient * status.soil_water_content * meteo.T + return nothing +end + +struct DomainSceneEvapotranspirationModel <: AbstractDomain_Scene_EvapotranspirationModel +end + +PlantSimEngine.inputs_(::DomainSceneEvapotranspirationModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneEvapotranspirationModel) = (evapotranspiration=0.0,) + +PlantSimEngine.dep(::DomainSceneEvapotranspirationModel) = ( + plant_transpiration=AllDomains(kind=:plant, process=:domain_plant_transpiration, policy=Integrate()), + soil_evaporation=AllDomains(kind=:soil, process=:domain_soil_evaporation, policy=Integrate()), +) + +function PlantSimEngine.run!(::DomainSceneEvapotranspirationModel, models, status, meteo, constants=nothing, extra=nothing) + plant_values = dependency_values(extra, :plant_transpiration, :transpiration) + soil_values = dependency_values(extra, :soil_evaporation, :evaporation) + status.evapotranspiration = sum(filter(x -> !isnothing(x), plant_values)) + sum(filter(x -> !isnothing(x), soil_values)) + return nothing +end + +struct DomainScenePlantEvapotranspirationModel <: AbstractDomain_Scene_Plant_EvapotranspirationModel +end + +PlantSimEngine.inputs_(::DomainScenePlantEvapotranspirationModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainScenePlantEvapotranspirationModel) = (plant_evapotranspiration=0.0,) + +PlantSimEngine.dep(::DomainScenePlantEvapotranspirationModel) = ( + plant_transpiration=AllDomains(kind=:plant, process=:domain_plant_transpiration, policy=Integrate()), +) + +function PlantSimEngine.run!(::DomainScenePlantEvapotranspirationModel, models, status, meteo, constants=nothing, extra=nothing) + plant_values = dependency_values(extra, :plant_transpiration, :transpiration) + status.plant_evapotranspiration = sum(filter(x -> !isnothing(x), plant_values)) + return nothing +end + +struct DomainHardLeafConductanceModel <: AbstractDomain_Hard_Leaf_ConductanceModel +end + +PlantSimEngine.inputs_(::DomainHardLeafConductanceModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainHardLeafConductanceModel) = (conductance=0.0,) + +function PlantSimEngine.run!(::DomainHardLeafConductanceModel, models, status, meteo, constants=nothing, extra=nothing) + status.conductance = 2.0 + return nothing +end + +struct DomainHardLeafEnergyModel <: AbstractDomain_Hard_Leaf_EnergyModel +end + +PlantSimEngine.dep(::DomainHardLeafEnergyModel) = (domain_hard_leaf_conductance=AbstractDomain_Hard_Leaf_ConductanceModel,) +PlantSimEngine.inputs_(::DomainHardLeafEnergyModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainHardLeafEnergyModel) = (leaf_temperature=0.0,) + +function PlantSimEngine.run!(::DomainHardLeafEnergyModel, models, status, meteo, constants=nothing, extra=nothing) + run_target!(models, status, :domain_hard_leaf_conductance; meteo=meteo, constants=constants, extra=extra) + status.leaf_temperature = 20.0 + status.conductance + return nothing +end + +struct DomainSceneConductanceSumModel <: AbstractDomain_Scene_Conductance_SumModel +end + +PlantSimEngine.inputs_(::DomainSceneConductanceSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneConductanceSumModel) = (conductance_sum=0.0,) + +PlantSimEngine.dep(::DomainSceneConductanceSumModel) = ( + conductance=AllDomains(kind=:plant, process=:domain_hard_leaf_conductance, var=:conductance, policy=Integrate()), +) + +function PlantSimEngine.run!(::DomainSceneConductanceSumModel, models, status, meteo, constants=nothing, extra=nothing) + conductance_values = dependency_values(extra, :conductance) + status.conductance_sum = sum(filter(x -> !isnothing(x), conductance_values)) + return nothing +end + +struct DomainHardTargetSignalModel{T} <: AbstractDomain_Hard_Target_SignalModel + coefficient::T +end + +PlantSimEngine.inputs_(::DomainHardTargetSignalModel) = (call_count=0,) +PlantSimEngine.outputs_(::DomainHardTargetSignalModel) = (call_count=0, signal=0.0,) + +function PlantSimEngine.run!(model::DomainHardTargetSignalModel, models, status, meteo, constants=nothing, extra=nothing) + status.call_count += 1 + status.signal = model.coefficient * status.call_count + return nothing +end + +struct DomainSceneHardTargetSumModel <: AbstractDomain_Scene_Hard_Target_SumModel end + +PlantSimEngine.inputs_(::DomainSceneHardTargetSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneHardTargetSumModel) = (hard_target_total=0.0,) + +PlantSimEngine.dep(::DomainSceneHardTargetSumModel) = ( + plant_signal=HardDomains(kind=:plant, process=:domain_hard_target_signal), +) + +function PlantSimEngine.run!(::DomainSceneHardTargetSumModel, models, status, meteo, constants=nothing, extra=nothing) + targets = dependency_targets(extra, :plant_signal) + for target in targets + run_target!(target) + run_target!(target) + end + status.hard_target_total = sum(target.status.signal for target in targets) + return nothing +end + +struct DomainSceneCallsTargetSumModel <: AbstractDomain_Scene_Calls_Target_SumModel end + +PlantSimEngine.inputs_(::DomainSceneCallsTargetSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneCallsTargetSumModel) = (calls_target_total=0.0,) + +function PlantSimEngine.run!(::DomainSceneCallsTargetSumModel, models, status, meteo, constants=nothing, extra=nothing) + targets = dependency_targets(extra, :plant_signal) + for target in targets + run_call!(target) + run_call!(target) + end + status.calls_target_total = sum(target.status.signal for target in targets) + return nothing +end + +struct DomainHardTargetLeafCounterModel{T} <: AbstractDomain_Hard_Target_Leaf_CounterModel + coefficient::T +end + +PlantSimEngine.inputs_(::DomainHardTargetLeafCounterModel) = (call_count=0,) +PlantSimEngine.outputs_(::DomainHardTargetLeafCounterModel) = (call_count=0, leaf_signal=0.0,) + +function PlantSimEngine.run!(model::DomainHardTargetLeafCounterModel, models, status, meteo, constants=nothing, extra=nothing) + status.call_count += 1 + status.leaf_signal = model.coefficient * status.call_count + return nothing +end + +struct DomainSceneHardTargetLeafSumModel <: AbstractDomain_Scene_Hard_Target_Leaf_SumModel end + +PlantSimEngine.inputs_(::DomainSceneHardTargetLeafSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneHardTargetLeafSumModel) = (leaf_hard_target_total=0.0,) + +PlantSimEngine.dep(::DomainSceneHardTargetLeafSumModel) = ( + leaf_calls=HardDomains(kind=:plant, scale=:Leaf, process=:domain_hard_target_leaf_counter), +) + +function PlantSimEngine.run!(::DomainSceneHardTargetLeafSumModel, models, status, meteo, constants=nothing, extra=nothing) + targets = dependency_targets(extra, :leaf_calls) + for target in targets + run_target!(target; publish=true) + end + status.leaf_hard_target_total = sum(target.status.leaf_signal for target in targets) + return nothing +end + +struct DomainSceneRoutedVectorModel <: AbstractDomain_Scene_Routed_VectorModel end + +PlantSimEngine.inputs_(::DomainSceneRoutedVectorModel) = (plant_transpirations=Float64[],) +PlantSimEngine.outputs_(::DomainSceneRoutedVectorModel) = (routed_total=0.0,) + +function PlantSimEngine.run!(::DomainSceneRoutedVectorModel, models, status, meteo, constants=nothing, extra=nothing) + status.routed_total = sum(status.plant_transpirations) + return nothing +end + +struct DomainSceneRoutedAggregateModel <: AbstractDomain_Scene_Routed_AggregateModel end + +PlantSimEngine.inputs_(::DomainSceneRoutedAggregateModel) = (daily_plant_transpiration=0.0,) +PlantSimEngine.outputs_(::DomainSceneRoutedAggregateModel) = (daily_routed_total=0.0,) + +function PlantSimEngine.run!(::DomainSceneRoutedAggregateModel, models, status, meteo, constants=nothing, extra=nothing) + status.daily_routed_total = status.daily_plant_transpiration + return nothing +end + +struct DomainMTGLeafFluxModel{T} <: AbstractDomain_Mtg_Leaf_FluxModel + coefficient::T +end + +PlantSimEngine.inputs_(::DomainMTGLeafFluxModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainMTGLeafFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(model::DomainMTGLeafFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = model.coefficient + return nothing +end + +struct DomainMTGLeafSoilFluxModel{T} <: AbstractDomain_Mtg_Leaf_Soil_FluxModel + coefficient::T +end + +PlantSimEngine.inputs_(::DomainMTGLeafSoilFluxModel) = (soil_signal=0.0,) +PlantSimEngine.outputs_(::DomainMTGLeafSoilFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(model::DomainMTGLeafSoilFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = model.coefficient * status.soil_signal + return nothing +end + +struct DomainGrowthLeafEmergenceModel <: AbstractDomain_Growth_Leaf_EmergenceModel end + +PlantSimEngine.inputs_(::DomainGrowthLeafEmergenceModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainGrowthLeafEmergenceModel) = (grown_leaves=0.0,) + +function PlantSimEngine.run!(::DomainGrowthLeafEmergenceModel, models, status, meteo, constants=nothing, extra=nothing) + if length(PlantSimEngine.status(extra)[:Leaf]) == 0 + add_organ!(status.node, extra, "+", :Leaf, 2; check=true) + end + status.grown_leaves = length(PlantSimEngine.status(extra)[:Leaf]) + return nothing +end + +struct DomainGrowthLeafFluxModel{T} <: AbstractDomain_Growth_Leaf_FluxModel + coefficient::T +end + +PlantSimEngine.inputs_(::DomainGrowthLeafFluxModel) = (grown_leaves=0.0,) +PlantSimEngine.outputs_(::DomainGrowthLeafFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(model::DomainGrowthLeafFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = model.coefficient * status.grown_leaves + return nothing +end + +struct DomainGrowthIntegratedFluxModel <: AbstractDomain_Growth_Integrated_FluxModel end + +PlantSimEngine.inputs_(::DomainGrowthIntegratedFluxModel) = (leaf_flux=-Inf,) +PlantSimEngine.outputs_(::DomainGrowthIntegratedFluxModel) = (integrated_leaf_flux=0.0,) + +function PlantSimEngine.run!(::DomainGrowthIntegratedFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.integrated_leaf_flux = sum(status.leaf_flux) + return nothing +end + +struct DomainSceneDependencyFluxSumModel <: AbstractDomain_Scene_Dependency_Flux_SumModel end + +PlantSimEngine.inputs_(::DomainSceneDependencyFluxSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneDependencyFluxSumModel) = (dependency_total=0.0, grouped_dependency_total=0.0,) + +PlantSimEngine.dep(::DomainSceneDependencyFluxSumModel) = ( + leaf_fluxes=AllDomains(kind=:plant, scale=:Leaf, process=:domain_mtg_leaf_flux, var=:leaf_flux), +) + +function PlantSimEngine.run!(::DomainSceneDependencyFluxSumModel, models, status, meteo, constants=nothing, extra=nothing) + grouped_values = dependency_values(extra, :leaf_fluxes) + flattened_values = dependency_values(extra, :leaf_fluxes; flatten=true) + status.grouped_dependency_total = sum(sum, grouped_values) + status.dependency_total = sum(flattened_values) + return nothing +end + +struct DomainSceneGrowthFluxSumModel <: AbstractDomain_Scene_Growth_Flux_SumModel end + +PlantSimEngine.inputs_(::DomainSceneGrowthFluxSumModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSceneGrowthFluxSumModel) = (growth_flux_total=0.0,) + +PlantSimEngine.dep(::DomainSceneGrowthFluxSumModel) = ( + leaf_fluxes=AllDomains(kind=:plant, scale=:Leaf, process=:domain_growth_leaf_flux, var=:leaf_flux), +) + +function PlantSimEngine.run!(::DomainSceneGrowthFluxSumModel, models, status, meteo, constants=nothing, extra=nothing) + status.growth_flux_total = sum(dependency_values(extra, :leaf_fluxes; flatten=true)) + return nothing +end + +struct DomainUpdateAllocationModel <: AbstractDomain_Update_AllocationModel end + +PlantSimEngine.inputs_(::DomainUpdateAllocationModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainUpdateAllocationModel) = (leaf_biomass=0.0,) + +function PlantSimEngine.run!(::DomainUpdateAllocationModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_biomass = 10.0 + return nothing +end + +struct DomainUpdatePruningModel <: AbstractDomain_Update_PruningModel end + +PlantSimEngine.inputs_(::DomainUpdatePruningModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainUpdatePruningModel) = (leaf_biomass=0.0,) + +function PlantSimEngine.run!(::DomainUpdatePruningModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_biomass = 0.0 + return nothing +end + +struct DomainUpdateObserverModel <: AbstractDomain_Update_ObserverModel end + +PlantSimEngine.inputs_(::DomainUpdateObserverModel) = (leaf_biomass=0.0,) +PlantSimEngine.outputs_(::DomainUpdateObserverModel) = (observed_biomass=0.0,) + +function PlantSimEngine.run!(::DomainUpdateObserverModel, models, status, meteo, constants=nothing, extra=nothing) + status.observed_biomass = status.leaf_biomass + return nothing +end + +struct DomainRemovalLeafFluxModel <: AbstractDomain_Removal_Leaf_FluxModel end + +PlantSimEngine.inputs_(::DomainRemovalLeafFluxModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainRemovalLeafFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(::DomainRemovalLeafFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = 1.0 + return nothing +end + +struct DomainRemovalPruningModel <: AbstractDomain_Removal_PruningModel end + +PlantSimEngine.inputs_(::DomainRemovalPruningModel) = (leaf_flux=Float64[], removed_count=0, removed_node_id=0,) +PlantSimEngine.outputs_(::DomainRemovalPruningModel) = (remaining_leaf_flux=0.0, removed_count=0, removed_node_id=0,) + +function PlantSimEngine.run!(::DomainRemovalPruningModel, models, status, meteo, constants=nothing, extra=nothing) + if status.removed_count == 0 && length(status.leaf_flux) > 1 + leaf_status = first(PlantSimEngine.status(extra)[:Leaf]) + status.removed_node_id = node_id(leaf_status.node) + remove_organ!(leaf_status.node, extra) + status.removed_count = 1 + end + status.remaining_leaf_flux = sum(status.leaf_flux) + return nothing +end + +struct DomainChurnLeafFluxModel <: AbstractDomain_Churn_Leaf_FluxModel end + +PlantSimEngine.inputs_(::DomainChurnLeafFluxModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainChurnLeafFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(::DomainChurnLeafFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = 1.0 + return nothing +end + +struct DomainChurnLeafControllerModel <: AbstractDomain_Churn_Leaf_ControllerModel end + +PlantSimEngine.inputs_(::DomainChurnLeafControllerModel) = ( + leaf_flux=Float64[], + created_count=0, + removed_count=0, + active_leaf_count=0, + last_removed_node_id=0, +) +PlantSimEngine.outputs_(::DomainChurnLeafControllerModel) = ( + created_count=0, + removed_count=0, + active_leaf_count=0, + last_removed_node_id=0, +) + +function PlantSimEngine.run!(::DomainChurnLeafControllerModel, models, status, meteo, constants=nothing, extra=nothing) + if isempty(status.leaf_flux) + add_organ!(status.node, extra, "+", :Leaf, 2; check=true) + status.created_count += 1 + else + leaf_status = first(PlantSimEngine.status(extra)[:Leaf]) + status.last_removed_node_id = node_id(leaf_status.node) + remove_organ!(leaf_status.node, extra) + status.removed_count += 1 + end + status.active_leaf_count = length(status.leaf_flux) + return nothing +end + +struct DomainSubtreeInternodeFluxModel <: AbstractDomain_Subtree_Internode_FluxModel end +struct DomainSubtreeLeafFluxModel <: AbstractDomain_Subtree_Leaf_FluxModel end + +PlantSimEngine.inputs_(::DomainSubtreeInternodeFluxModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSubtreeInternodeFluxModel) = (internode_flux=0.0,) + +function PlantSimEngine.run!(::DomainSubtreeInternodeFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.internode_flux = 2.0 + return nothing +end + +PlantSimEngine.inputs_(::DomainSubtreeLeafFluxModel) = NamedTuple() +PlantSimEngine.outputs_(::DomainSubtreeLeafFluxModel) = (leaf_flux=0.0,) + +function PlantSimEngine.run!(::DomainSubtreeLeafFluxModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_flux = 1.0 + return nothing +end + +struct DomainSubtreePruningModel <: AbstractDomain_Subtree_PruningModel end + +PlantSimEngine.inputs_(::DomainSubtreePruningModel) = ( + internode_flux=Float64[], + leaf_flux=Float64[], + removed_count=0, + removed_internode_id=0, + removed_leaf_id=0, + remaining_internode_count=0, + remaining_leaf_count=0, +) +PlantSimEngine.outputs_(::DomainSubtreePruningModel) = ( + removed_count=0, + removed_internode_id=0, + removed_leaf_id=0, + remaining_internode_count=0, + remaining_leaf_count=0, +) + +function PlantSimEngine.run!(::DomainSubtreePruningModel, models, status, meteo, constants=nothing, extra=nothing) + graph_status = PlantSimEngine.status(extra) + if status.removed_count == 0 && !isempty(get(graph_status, :Internode, Status[])) + internode_status = first(graph_status[:Internode]) + leaf_status = first(graph_status[:Leaf]) + status.removed_internode_id = node_id(internode_status.node) + status.removed_leaf_id = node_id(leaf_status.node) + remove_organ!(internode_status.node, extra; recursive=true) + status.removed_count = 1 + end + status.remaining_internode_count = length(status.internode_flux) + status.remaining_leaf_count = length(status.leaf_flux) + return nothing +end + +struct DomainReparentLeafControllerModel <: AbstractDomain_Reparent_Leaf_ControllerModel end + +PlantSimEngine.inputs_(::DomainReparentLeafControllerModel) = ( + leaf_flux=Float64[], + reparented_count=0, + new_parent_id=0, + leaf_parent_id=0, + active_leaf_count=0, +) +PlantSimEngine.outputs_(::DomainReparentLeafControllerModel) = ( + reparented_count=0, + new_parent_id=0, + leaf_parent_id=0, + active_leaf_count=0, +) + +function PlantSimEngine.run!(::DomainReparentLeafControllerModel, models, status, meteo, constants=nothing, extra=nothing) + graph_status = PlantSimEngine.status(extra) + if status.reparented_count == 0 + leaf_status = only(graph_status[:Leaf]) + new_parent_status = graph_status[:Internode][2] + reparent_organ!(leaf_status.node, new_parent_status.node, extra) + status.reparented_count = 1 + status.new_parent_id = node_id(new_parent_status.node) + end + leaf_status = only(graph_status[:Leaf]) + status.leaf_parent_id = node_id(parent(leaf_status.node)) + status.active_leaf_count = length(status.leaf_flux) + return nothing +end + +@testset "Domain simulation: two plants, soil, and daily scene aggregation" begin + hourly_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:25 + ]) + + oil_palm_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.01)) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + + maize_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.3)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.02)) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + + soil_mapping = ModelMapping( + ModelSpec(DomainSoilWaterModel(0.35)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainSoilEvaporationModel(0.2)) |> TimeStepModel(Dates.Hour(1)), + status=(soil_water_content=0.0, evaporation=0.0), + ) + + scene_mapping = ModelMapping( + ModelSpec(DomainSceneEvapotranspirationModel()) |> TimeStepModel(Dates.Day(1)), + status=(evapotranspiration=0.0,), + ) + + simulation_mapping = SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:maize, maize_mapping; kind=:plant), + Domain(:soil, soil_mapping; kind=:soil), + Domain(:scene, scene_mapping; kind=:scene), + ) + + domain_rows = explain_domains(simulation_mapping) + @test length(domain_rows) == 4 + @test any(row -> row.domain == :oil_palm && row.kind == :plant, domain_rows) + + model_rows = explain_domain_models(simulation_mapping) + @test length(model_rows) == 7 + @test any(row -> row.domain == :oil_palm && row.process == :domain_absorbed_radiation && haskey(row.meteo_inputs, :Ri_PAR_f), model_rows) + + sim = run!(simulation_mapping, hourly_meteo, check=true) + @test status(sim, :oil_palm).transpiration ≈ 0.01 * 0.5 * 100.0 + @test outputs(sim) === sim.outputs + + schedule = explain_schedule(sim) + @test any(row -> row.domain == :scene && row.dt_seconds == 86_400.0, schedule) + @test any(row -> row.domain == :oil_palm && row.dt_seconds == 3_600.0, schedule) + + deps = explain_domain_dependencies(sim) + @test length(deps) == 3 + @test count(row -> row.dependency == :plant_transpiration, deps) == 2 + @test count(row -> row.dependency == :soil_evaporation, deps) == 1 + @test all(row -> isnothing(row.variable), deps) + + scene_key = DomainModelKey(:scene, :Default, :domain_scene_evapotranspiration) + scene_values = sim.outputs[(scene_key, :evapotranspiration)] + + # Dates.Day(1) currently aligns to step 1, then step 25 when the base step is hourly. + # The second scene value integrates producer values from steps 2:25. + hourly_plant_sum = 0.01 * 0.5 * 100.0 + 0.02 * 0.3 * 100.0 + hourly_soil = 0.2 * (0.35 - 0.001 * 20.0) * 20.0 + expected_daily_et = 24.0 * (hourly_plant_sum + hourly_soil) + + @test length(scene_values) == 2 + @test scene_values[2] ≈ expected_daily_et +end + +@testset "Domain simulation validation" begin + hourly_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + plant_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.01)) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + + raw_domain = Domain( + :plant_kw; + kind=:plant, + mapping=( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.01)) |> TimeStepModel(Dates.Hour(1)), + Status(absorbed_radiation=0.0, transpiration=0.0), + ), + ) + @test raw_domain.mapping isa ModelMapping + + @test_throws ErrorException SimulationMapping( + Domain(:plant, plant_mapping; kind=:plant), + Domain(:plant, plant_mapping; kind=:plant), + ) + + daily_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:25 + ]) + mixed_rate_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.01)) |> TimeStepModel(Dates.Day(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + mixed_sim = run!( + SimulationMapping(Domain(:mixed_plant, mixed_rate_mapping; kind=:plant)), + daily_meteo, + check=true, + ) + absorbed_key = DomainModelKey(:mixed_plant, :Default, :domain_absorbed_radiation) + transpiration_key = DomainModelKey(:mixed_plant, :Default, :domain_plant_transpiration) + @test length(mixed_sim.outputs[(absorbed_key, :absorbed_radiation)]) == 25 + @test length(mixed_sim.outputs[(transpiration_key, :transpiration)]) == 2 + + unmatched_scene_mapping = ModelMapping( + ModelSpec(DomainSceneEvapotranspirationModel()) |> TimeStepModel(Dates.Day(1)), + status=(evapotranspiration=0.0,), + ) + unmatched_error = try + run!( + SimulationMapping(Domain(:scene, unmatched_scene_mapping; kind=:scene)), + hourly_meteo, + check=true, + ) + "" + catch err + sprint(showerror, err) + end + @test occursin("Domain dependency `plant_transpiration`", unmatched_error) + @test occursin("consumer `scene/Default/domain_scene_evapotranspiration`", unmatched_error) + @test occursin("AllDomains(kind=:plant, process=:domain_plant_transpiration, policy=Integrate())", unmatched_error) + @test occursin("Available producers:", unmatched_error) + + multi_process_scene_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainScenePlantEvapotranspirationModel()) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, plant_evapotranspiration=0.0), + ) + multi_scene_sim = run!( + SimulationMapping( + Domain(:plant, plant_mapping; kind=:plant), + Domain(:scene, multi_process_scene_mapping; kind=:scene), + ), + hourly_meteo, + check=true, + ) + @test status(multi_scene_sim, :scene).plant_evapotranspiration > 0.0 + + hard_plant_mapping = ModelMapping( + ModelSpec(DomainHardLeafConductanceModel()) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainHardLeafEnergyModel()) |> TimeStepModel(Dates.Hour(1)), + status=(conductance=0.0, leaf_temperature=0.0), + ) + conductance_scene_mapping = ModelMapping( + ModelSpec(DomainSceneConductanceSumModel()) |> TimeStepModel(Dates.Hour(1)), + status=(conductance_sum=0.0,), + ) + hard_sim = run!( + SimulationMapping( + Domain(:hard_plant, hard_plant_mapping; kind=:plant), + Domain(:scene, conductance_scene_mapping; kind=:scene), + ), + hourly_meteo, + check=true, + ) + conductance_key = DomainModelKey(:hard_plant, :Default, :domain_hard_leaf_conductance) + @test hard_sim.outputs[(conductance_key, :conductance)] == [2.0, 2.0] + @test status(hard_sim, :scene).conductance_sum == 2.0 + hard_deps = explain_domain_dependencies(hard_sim) + @test only(hard_deps).variable == :conductance + + hard_target_plant_mapping = ModelMapping( + ModelSpec(DomainHardTargetSignalModel(2.0)) |> TimeStepModel(Dates.Hour(1)), + status=(call_count=0, signal=0.0), + ) + hard_target_scene_mapping = ModelMapping( + ModelSpec(DomainSceneHardTargetSumModel()) |> TimeStepModel(Dates.Hour(1)), + status=(hard_target_total=0.0,), + ) + hard_target_sim = run!( + SimulationMapping( + Domain(:hard_target_plant, hard_target_plant_mapping; kind=:plant), + Domain(:scene, hard_target_scene_mapping; kind=:scene), + ), + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)), + check=true, + ) + @test status(hard_target_sim, :hard_target_plant).call_count == 2 + @test status(hard_target_sim, :hard_target_plant).signal ≈ 4.0 + @test status(hard_target_sim, :scene).hard_target_total ≈ 4.0 + @test only(explain_domain_dependencies(hard_target_sim)).mode == :hard_domain + + calls_target_scene_mapping = ModelMapping( + ModelSpec(DomainSceneCallsTargetSumModel()) |> + Calls(:plant_signal => Many(kind=:plant, process=:domain_hard_target_signal)) |> + TimeStep(Dates.Hour(1)), + status=(calls_target_total=0.0,), + ) + calls_target_sim = run!( + SimulationMapping( + Domain(:hard_target_plant, hard_target_plant_mapping; kind=:plant), + Domain(:scene, calls_target_scene_mapping; kind=:scene), + ), + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)), + check=true, + ) + @test status(calls_target_sim, :hard_target_plant).call_count == 2 + @test status(calls_target_sim, :hard_target_plant).signal ≈ 4.0 + @test status(calls_target_sim, :scene).calls_target_total ≈ 4.0 + @test only(explain_domain_dependencies(calls_target_sim)).mode == :hard_domain + + route_source = AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:transpiration) + bad_route_source = AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:missing_output) + route_selector_error = try + run!( + SimulationMapping( + Domain(:plant, plant_mapping; kind=:plant), + Domain(:scene, multi_process_scene_mapping; kind=:scene); + routes=(Route( + from=bad_route_source, + to=DomainRouteTarget(:scene, var=:plant_transpirations), + ),), + ), + hourly_meteo, + check=true, + ) + "" + catch err + sprint(showerror, err) + end + @test occursin("Route 1 from `AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:missing_output)`", route_selector_error) + @test occursin("Models matching all selector fields except `var=:missing_output`", route_selector_error) + @test occursin("plant/Default/domain_plant_transpiration outputs=(:transpiration)", route_selector_error) + + missing_target_scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedAggregateModel()) |> TimeStepModel(Dates.Hour(1)), + status=(daily_plant_transpiration=0.0, daily_routed_total=0.0), + ) + @test_throws "does not contain variable `plant_transpirations`" run!( + SimulationMapping( + Domain(:plant, plant_mapping; kind=:plant), + Domain(:scene, missing_target_scene_mapping; kind=:scene); + routes=(Route( + from=route_source, + to=DomainRouteTarget(:scene, var=:plant_transpirations), + ),), + ), + hourly_meteo, + check=true, + ) + + wrong_process_scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedAggregateModel()) |> TimeStepModel(Dates.Hour(1)), + status=(plant_transpirations=[0.0], daily_plant_transpiration=0.0, daily_routed_total=0.0), + ) + @test_throws "does not consume variable `plant_transpirations`" run!( + SimulationMapping( + Domain(:plant, plant_mapping; kind=:plant), + Domain(:scene, wrong_process_scene_mapping; kind=:scene); + routes=(Route( + from=route_source, + to=DomainRouteTarget(:scene, var=:plant_transpirations, process=:domain_scene_routed_aggregate), + ),), + ), + hourly_meteo, + check=true, + ) + + @test_throws "has a selector but uses a single-status ModelMapping" run!( + Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)), + SimulationMapping(Domain(:plant, plant_mapping; kind=:plant, selector=:Plant)), + hourly_meteo, + check=true, + ) +end + +@testset "Domain simulation routes" begin + hourly_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:25 + ]) + + oil_palm_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.01)) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + + maize_mapping = ModelMapping( + ModelSpec(DomainAbsorbedRadiationModel(0.3)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainPlantTranspirationModel(0.02)) |> TimeStepModel(Dates.Hour(1)), + status=(absorbed_radiation=0.0, transpiration=0.0), + ) + + hourly_scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedVectorModel()) |> TimeStepModel(Dates.Hour(1)), + status=(routed_total=0.0,), + ) + + vector_route = Route( + from=AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:transpiration), + to=DomainRouteTarget(:scene, var=:plant_transpirations, process=:domain_scene_routed_vector), + cardinality=ManyToOneVector(), + ) + + vector_sim = run!( + SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:maize, maize_mapping; kind=:plant), + Domain(:scene, hourly_scene_mapping; kind=:scene); + routes=(vector_route,), + ), + hourly_meteo, + check=true, + ) + hourly_plant_sum = 0.01 * 0.5 * 100.0 + 0.02 * 0.3 * 100.0 + @test :plant_transpirations in propertynames(status(vector_sim, :scene)) + @test status(vector_sim, :scene).plant_transpirations ≈ [0.5, 0.6] + @test status(vector_sim, :scene).routed_total ≈ hourly_plant_sum + + route_rows = explain_routes(vector_sim) + @test length(route_rows) == 2 + @test all(row -> row.target_var == :plant_transpirations, route_rows) + @test all(row -> row.cardinality == ManyToOneVector, route_rows) + + inputs_route_scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedVectorModel()) |> + Inputs(:plant_transpirations => Many(kind=:plant, process=:domain_plant_transpiration, var=:transpiration)) |> + TimeStep(Dates.Hour(1)), + status=(routed_total=0.0,), + ) + inputs_route_sim = run!( + SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:maize, maize_mapping; kind=:plant), + Domain(:scene, inputs_route_scene_mapping; kind=:scene), + ), + hourly_meteo, + check=true, + ) + @test :plant_transpirations in propertynames(status(inputs_route_sim, :scene)) + @test status(inputs_route_sim, :scene).plant_transpirations ≈ [0.5, 0.6] + @test status(inputs_route_sim, :scene).routed_total ≈ hourly_plant_sum + input_route_rows = explain_routes(inputs_route_sim) + @test length(input_route_rows) == 2 + @test all(row -> row.target_var == :plant_transpirations, input_route_rows) + @test all(row -> row.cardinality == ManyToOneVector, input_route_rows) + + reordered_collector_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedVectorModel()) |> TimeStepModel(Dates.Hour(1)), + status=(plant_transpirations=[0.0], routed_total=0.0), + ) + reordered_route = Route( + from=AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:transpiration), + to=DomainRouteTarget(:collector, var=:plant_transpirations, process=:domain_scene_routed_vector), + cardinality=ManyToOneVector(), + ) + reordered_sim = run!( + SimulationMapping( + Domain(:collector, reordered_collector_mapping; kind=:soil), + Domain(:oil_palm, oil_palm_mapping; kind=:plant); + routes=(reordered_route,), + ), + hourly_meteo, + check=true, + ) + @test status(reordered_sim, :collector).plant_transpirations ≈ [0.5] + @test status(reordered_sim, :collector).routed_total ≈ 0.5 + + cyclic_scene_source_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedVectorModel()) |> TimeStepModel(Dates.Hour(1)), + status=(plant_transpirations=[0.0], routed_total=0.0), + ) + cyclic_route = Route( + from=AllDomains(kind=:scene, process=:domain_scene_routed_vector, var=:routed_total), + to=DomainRouteTarget(:oil_palm, var=:absorbed_radiation, process=:domain_plant_transpiration), + cardinality=ManyToOneAggregate(sum), + ) + @test_throws "Cyclic domain run-order constraints" run!( + SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:scene, cyclic_scene_source_mapping; kind=:scene); + routes=(cyclic_route,), + ), + hourly_meteo, + check=true, + ) + + daily_scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedAggregateModel()) |> TimeStepModel(Dates.Day(1)), + status=(daily_plant_transpiration=0.0, daily_routed_total=0.0), + ) + + aggregate_route = Route( + from=AllDomains(kind=:plant, process=:domain_plant_transpiration, var=:transpiration), + to=DomainRouteTarget(:scene, var=:daily_plant_transpiration, process=:domain_scene_routed_aggregate), + cardinality=ManyToOneAggregate(sum), + policy=Integrate(), + ) + + aggregate_sim = run!( + SimulationMapping( + Domain(:oil_palm, oil_palm_mapping; kind=:plant), + Domain(:maize, maize_mapping; kind=:plant), + Domain(:scene, daily_scene_mapping; kind=:scene); + routes=(aggregate_route,), + ), + hourly_meteo, + check=true, + ) + @test status(aggregate_sim, :scene).daily_plant_transpiration ≈ 24.0 * hourly_plant_sum + @test status(aggregate_sim, :scene).daily_routed_total ≈ 24.0 * hourly_plant_sum + + aggregate_rows = explain_routes(aggregate_sim) + @test length(aggregate_rows) == 2 + @test all(row -> row.dt_seconds == 86_400.0, aggregate_rows) + @test all(row -> row.cardinality <: ManyToOneAggregate, aggregate_rows) +end + +@testset "Domain graph dependency policy with changing topology" begin + first_step = PlantSimEngine.DomainNodeValues([11, 12], Any[1.0, 2.0]) + second_step = PlantSimEngine.DomainNodeValues([11, 13], Any[3.0, 4.0]) + + integrated = PlantSimEngine._apply_dependency_policy(Any[first_step, second_step], Integrate()) + @test integrated.ids == [11, 12, 13] + @test integrated.values ≈ [4.0, 2.0, 4.0] + + aggregated = PlantSimEngine._apply_dependency_policy(Any[first_step, second_step], Aggregate()) + @test aggregated.ids == [11, 12, 13] + @test aggregated.values ≈ [2.0, 2.0, 4.0] + + no_ids_first_step = PlantSimEngine.DomainNodeValues(Any[1.0, 2.0]) + no_ids_second_step = PlantSimEngine.DomainNodeValues(Any[3.0]) + @test_throws "changing vector lengths" PlantSimEngine._apply_dependency_policy( + Any[no_ids_first_step, no_ids_second_step], + Integrate(), + ) +end + +@testset "Staged MTG-backed domain simulation" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + oil_palm = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + oil_axis = Node(oil_palm, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + Node(oil_axis, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 3)) + maize = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 2, 1)) + maize_axis = Node(maize, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + Node(maize_axis, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 3)) + + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)), + Atmosphere(T=25.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)), + ]) + + oil_leaf_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainMTGLeafFluxModel(0.5)) |> TimeStepModel(Dates.Hour(1)), + ), + ) + maize_leaf_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainMTGLeafFluxModel(0.7)) |> TimeStepModel(Dates.Hour(1)), + ), + ) + scene_mapping = ModelMapping( + ModelSpec(DomainSceneRoutedVectorModel()) |> TimeStepModel(Dates.Hour(1)), + status=(plant_transpirations=[0.0], routed_total=0.0), + ) + route = Route( + from=AllDomains(kind=:plant, scale=:Leaf, process=:domain_mtg_leaf_flux, var=:leaf_flux), + to=DomainRouteTarget(:scene, var=:plant_transpirations, process=:domain_scene_routed_vector), + cardinality=ManyToOneVector(), + ) + + sim = run!( + scene, + SimulationMapping( + Domain(:oil_palm, oil_leaf_mapping; kind=:plant, selector=oil_palm), + Domain(:maize, maize_leaf_mapping; kind=:plant, selector=maize), + Domain(:scene, scene_mapping; kind=:scene); + routes=(route,), + ), + meteo, + check=true, + ) + + @test length(status(sim, :oil_palm, :Leaf)) == 1 + @test length(status(sim, :maize, :Leaf)) == 1 + @test length(status(sim, :Leaf)) == 2 + @test length(status(sim, :Default)) == 1 + @test status(sim, :scene).plant_transpirations ≈ [0.5, 0.7] + @test status(sim, :scene).routed_total ≈ 1.2 + @test sim.outputs[(DomainModelKey(:scene, :Default, :domain_scene_routed_vector), :routed_total)] ≈ [1.2, 1.2] + @test sim.outputs[(DomainModelKey(:oil_palm, :Leaf, :domain_mtg_leaf_flux), :leaf_flux)] == [[0.5], [0.5]] + @test sim.outputs[(DomainModelKey(:maize, :Leaf, :domain_mtg_leaf_flux), :leaf_flux)] == [[0.7], [0.7]] + status_rows = explain_domain_statuses(sim) + @test sum(row -> row.scale == :Leaf, status_rows) == 2 + @test only(row.nstatuses for row in status_rows if row.domain == :scene && row.scale == :Default) == 1 + + forest_leaf_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainMTGLeafFluxModel(0.4)) |> TimeStepModel(Dates.Hour(1)), + ), + ) + forest_sim = run!( + scene, + SimulationMapping( + Domain(:forest, forest_leaf_mapping; kind=:plant, selector=:Plant), + Domain(:scene, scene_mapping; kind=:scene); + routes=(route,), + ), + meteo, + check=true, + ) + @test length(status(forest_sim, :forest, :Leaf)) == 2 + @test length(status(forest_sim, :Leaf)) == 2 + @test status(forest_sim, :scene).plant_transpirations ≈ [0.4, 0.4] + @test status(forest_sim, :scene).routed_total ≈ 0.8 + @test forest_sim.outputs[(DomainModelKey(:forest, :Leaf, :domain_mtg_leaf_flux), :leaf_flux)] == [[0.4, 0.4], [0.4, 0.4]] + @test only(row.nstatuses for row in explain_domain_statuses(forest_sim) if row.domain == :forest && row.scale == :Leaf) == 2 + + @test_throws "matched overlapping MTG roots" run!( + scene, + SimulationMapping( + Domain( + :overlapping_forest, + forest_leaf_mapping; + kind=:plant, + selector=node -> symbol(node) in (:Plant, :Leaf), + ), + ), + meteo, + check=true, + ) + + dependency_scene_mapping = ModelMapping( + ModelSpec(DomainSceneDependencyFluxSumModel()) |> TimeStepModel(Dates.Hour(1)), + status=(dependency_total=0.0, grouped_dependency_total=0.0), + ) + dependency_sim = run!( + scene, + SimulationMapping( + Domain(:oil_palm, oil_leaf_mapping; kind=:plant, selector=oil_palm), + Domain(:maize, maize_leaf_mapping; kind=:plant, selector=maize), + Domain(:scene, dependency_scene_mapping; kind=:scene), + ), + meteo, + check=true, + ) + @test status(dependency_sim, :scene).dependency_total ≈ 1.2 + @test status(dependency_sim, :scene).grouped_dependency_total ≈ 1.2 + + soil_mapping = ModelMapping( + ModelSpec(DomainSoilWaterModel(0.35)) |> TimeStepModel(Dates.Hour(1)), + status=(soil_water_content=0.0,), + ) + soil_leaf_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainMTGLeafSoilFluxModel(2.0)) |> TimeStepModel(Dates.Hour(1)), + ), + ) + soil_route = Route( + from=AllDomains(kind=:soil, process=:domain_soil_water, var=:soil_water_content), + to=DomainRouteTarget(:oil_palm, scale=:Leaf, var=:soil_signal, process=:domain_mtg_leaf_soil_flux), + cardinality=OneToManyBroadcast(), + ) + soil_to_graph_sim = run!( + scene, + SimulationMapping( + Domain(:soil, soil_mapping; kind=:soil), + Domain(:oil_palm, soil_leaf_mapping; kind=:plant, selector=oil_palm); + routes=(soil_route,), + ), + meteo, + check=true, + ) + expected_soil_signals = [0.35 - 0.001 * 20.0, 0.35 - 0.001 * 25.0] + @test only(status(soil_to_graph_sim, :oil_palm, :Leaf)).soil_signal ≈ expected_soil_signals[2] + @test soil_to_graph_sim.outputs[(DomainModelKey(:oil_palm, :Leaf, :domain_mtg_leaf_soil_flux), :leaf_flux)] == [[2.0 * expected_soil_signals[1]], [2.0 * expected_soil_signals[2]]] + + reordered_soil_to_graph_sim = run!( + scene, + SimulationMapping( + Domain(:oil_palm, soil_leaf_mapping; kind=:plant, selector=oil_palm), + Domain(:soil, soil_mapping; kind=:soil); + routes=(soil_route,), + ), + meteo, + check=true, + ) + @test only(status(reordered_soil_to_graph_sim, :oil_palm, :Leaf)).soil_signal ≈ expected_soil_signals[2] + @test reordered_soil_to_graph_sim.outputs[(DomainModelKey(:oil_palm, :Leaf, :domain_mtg_leaf_soil_flux), :leaf_flux)] == [[2.0 * expected_soil_signals[1]], [2.0 * expected_soil_signals[2]]] +end + +@testset "Hard-domain targets from MTG-backed domains" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 2, 2)) + + meteo = Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + plant_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainHardTargetLeafCounterModel(1.5)) |> TimeStepModel(Dates.Hour(1)), + Status(call_count=0, leaf_signal=0.0), + ), + ) + scene_mapping = ModelMapping( + ModelSpec(DomainSceneHardTargetLeafSumModel()) |> TimeStepModel(Dates.Hour(1)), + status=(leaf_hard_target_total=0.0,), + ) + + sim = run!( + scene, + SimulationMapping( + Domain(:hard_target_plant, plant_mapping; kind=:plant, selector=plant), + Domain(:scene, scene_mapping; kind=:scene), + ), + meteo, + check=true, + ) + + leaf_statuses = status(sim, :hard_target_plant, :Leaf) + @test length(leaf_statuses) == 2 + @test all(st -> st.call_count == 1, leaf_statuses) + @test all(st -> st.leaf_signal ≈ 1.5, leaf_statuses) + @test status(sim, :scene).leaf_hard_target_total ≈ 3.0 + @test sim.outputs[(DomainModelKey(:hard_target_plant, :Leaf, :domain_hard_target_leaf_counter), :leaf_signal)] == [1.5, 1.5] +end + +@testset "MTG-backed domain growth registration" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + growth_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainGrowthLeafEmergenceModel()) |> TimeStepModel(Dates.Hour(1)), + ), + :Leaf => ( + ModelSpec(DomainGrowthLeafFluxModel(0.9)) |> + MultiScaleModel([:grown_leaves => (:Plant => :grown_leaves)]) |> + TimeStepModel(Dates.Hour(1)), + ), + ) + scene_mapping = ModelMapping( + ModelSpec(DomainSceneGrowthFluxSumModel()) |> TimeStepModel(Dates.Hour(1)), + status=(growth_flux_total=0.0,), + ) + + sim = run!( + scene, + SimulationMapping( + Domain(:growing_plant, growth_mapping; kind=:plant, selector=plant), + Domain(:scene, scene_mapping; kind=:scene), + ), + meteo, + check=true, + ) + + @test length(status(sim, :growing_plant, :Leaf)) == 1 + @test length(status(sim, :Leaf)) == 1 + @test only(status(sim, :growing_plant, :Leaf)).grown_leaves == 1.0 + @test only(status(sim, :growing_plant, :Leaf)).leaf_flux ≈ 0.9 + @test status(sim, :scene).growth_flux_total ≈ 0.9 + @test sim.outputs[(DomainModelKey(:growing_plant, :Leaf, :domain_growth_leaf_flux), :leaf_flux)] == [[0.9], [0.9]] + @test sim.outputs[(DomainModelKey(:scene, :Default, :domain_scene_growth_flux_sum), :growth_flux_total)] ≈ [0.9, 0.9] + @test only(row.nstatuses for row in explain_domain_statuses(sim) if row.domain == :growing_plant && row.scale == :Leaf) == 1 + + multirate_scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + multirate_plant = Node(multirate_scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + multirate_meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:25 + ]) + multirate_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainGrowthLeafEmergenceModel()) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainGrowthIntegratedFluxModel()) |> + MultiScaleModel([:leaf_flux => [:Leaf]]) |> + InputBindings(; leaf_flux=(process=:domain_growth_leaf_flux, scale=:Leaf, var=:leaf_flux, policy=Integrate())) |> + TimeStepModel(Dates.Day(1)), + ), + :Leaf => ( + ModelSpec(DomainGrowthLeafFluxModel(0.9)) |> + MultiScaleModel([:grown_leaves => (:Plant => :grown_leaves)]) |> + TimeStepModel(Dates.Hour(1)), + ), + ) + multirate_sim = run!( + multirate_scene, + SimulationMapping(Domain(:growing_plant, multirate_mapping; kind=:plant, selector=multirate_plant)), + multirate_meteo, + check=true, + ) + + @test length(status(multirate_sim, :growing_plant, :Leaf)) == 1 + @test only(status(multirate_sim, :growing_plant, :Plant)).integrated_leaf_flux ≈ 24.0 * 0.9 + integrated_outputs = multirate_sim.outputs[(DomainModelKey(:growing_plant, :Plant, :domain_growth_integrated_flux), :integrated_leaf_flux)] + @test only.(integrated_outputs) ≈ [0.9, 24.0 * 0.9] + @test length(integrated_outputs) == 2 +end + +@testset "MTG-backed domain variable updates" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + update_mapping = ModelMapping( + :Leaf => ( + ModelSpec(DomainUpdateAllocationModel()) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainUpdatePruningModel()) |> + Updates(:leaf_biomass; after=:domain_update_allocation) |> + TimeStepModel(Dates.Hour(1)), + ModelSpec(DomainUpdateObserverModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + resolved_update_specs = resolved_model_specs(update_mapping) + inferred_update_binding = input_bindings(resolved_update_specs[:Leaf][:domain_update_observer]).leaf_biomass + @test inferred_update_binding.process == :domain_update_pruning + + sim = run!( + scene, + SimulationMapping(Domain(:updated_plant, update_mapping; kind=:plant, selector=plant)), + meteo, + check=true, + ) + + leaf_status = only(status(sim, :updated_plant, :Leaf)) + @test leaf_status.leaf_biomass == 0.0 + @test leaf_status.observed_biomass == 0.0 + @test sim.outputs[(DomainModelKey(:updated_plant, :Leaf, :domain_update_pruning), :leaf_biomass)] == [[0.0], [0.0]] + @test sim.outputs[(DomainModelKey(:updated_plant, :Leaf, :domain_update_observer), :observed_biomass)] == [[0.0], [0.0]] +end + +@testset "MTG-backed domain leaf removal registration" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 2, 2)) + + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + removal_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainRemovalPruningModel()) |> + MultiScaleModel([:leaf_flux => [:Leaf]]) |> + TimeStepModel(Dates.Hour(1)), + Status(removed_count=0, removed_node_id=0, remaining_leaf_flux=0.0), + ), + :Leaf => ( + ModelSpec(DomainRemovalLeafFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + sim = run!( + scene, + SimulationMapping(Domain(:pruned_plant, removal_mapping; kind=:plant, selector=plant)), + meteo, + check=true, + ) + + plant_status = only(status(sim, :pruned_plant, :Plant)) + @test length(status(sim, :pruned_plant, :Leaf)) == 1 + @test length(status(sim, :Leaf)) == 1 + @test length(plant_status.leaf_flux) == 1 + @test plant_status.removed_count == 1 + @test plant_status.removed_node_id > 0 + @test plant_status.remaining_leaf_flux ≈ 1.0 + @test sim.outputs[(DomainModelKey(:pruned_plant, :Leaf, :domain_removal_leaf_flux), :leaf_flux)] == [[1.0], [1.0]] + @test sim.outputs[(DomainModelKey(:pruned_plant, :Plant, :domain_removal_pruning), :remaining_leaf_flux)] == [[1.0], [1.0]] + @test_throws ErrorException remove_organ!(plant, only(sim.domain_states[:pruned_plant].simulations)) + + multirate_scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + multirate_plant = Node(multirate_scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + Node(multirate_plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + Node(multirate_plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 2, 2)) + multirate_removal_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainRemovalPruningModel()) |> + MultiScaleModel([:leaf_flux => [:Leaf]]) |> + InputBindings(; leaf_flux=(process=:domain_removal_leaf_flux, scale=:Leaf, var=:leaf_flux, policy=Integrate())) |> + TimeStepModel(Dates.Day(1)), + Status(removed_count=0, removed_node_id=0, remaining_leaf_flux=0.0), + ), + :Leaf => ( + ModelSpec(DomainRemovalLeafFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + multirate_sim = run!( + multirate_scene, + SimulationMapping(Domain(:pruned_plant, multirate_removal_mapping; kind=:plant, selector=multirate_plant)), + meteo, + check=true, + ) + + multirate_plant_status = only(status(multirate_sim, :pruned_plant, :Plant)) + multirate_graph = only(multirate_sim.domain_states[:pruned_plant].simulations) + @test length(status(multirate_sim, :pruned_plant, :Leaf)) == 1 + @test length(multirate_plant_status.leaf_flux) == 1 + @test multirate_plant_status.remaining_leaf_flux ≈ 1.0 + @test all(key -> key.node_id != multirate_plant_status.removed_node_id, keys(multirate_graph.temporal_state.streams)) + @test all(key -> key.node_id != multirate_plant_status.removed_node_id, keys(multirate_graph.temporal_state.caches)) +end + +@testset "MTG-backed domain repeated create/remove churn" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:4 + ]) + + churn_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainChurnLeafControllerModel()) |> + MultiScaleModel([:leaf_flux => [:Leaf]]) |> + InputBindings(; leaf_flux=(process=:domain_churn_leaf_flux, scale=:Leaf, var=:leaf_flux, policy=HoldLast())) |> + TimeStepModel(Dates.Hour(1)), + Status(created_count=0, removed_count=0, active_leaf_count=0, last_removed_node_id=0), + ), + :Leaf => ( + ModelSpec(DomainChurnLeafFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + sim = run!( + scene, + SimulationMapping(Domain(:churn_plant, churn_mapping; kind=:plant, selector=plant)), + meteo, + check=true, + ) + + plant_status = only(status(sim, :churn_plant, :Plant)) + graph_sim = only(sim.domain_states[:churn_plant].simulations) + @test plant_status.created_count == 2 + @test plant_status.removed_count == 2 + @test plant_status.active_leaf_count == 0 + @test length(plant_status.leaf_flux) == 0 + @test get(status(graph_sim), :Leaf, Status[]) == Status[] + @test get(status(sim.domain_states[:churn_plant]), :Leaf, Status[]) == Status[] + @test status(sim, :churn_plant, :Leaf) == Status[] + @test status(sim, :Leaf) == Status[] + @test only(row.nstatuses for row in explain_domain_statuses(sim) if row.domain == :churn_plant && row.scale == :Leaf) == 0 + @test all(key -> key.scale != :Leaf, keys(graph_sim.temporal_state.streams)) + @test all(key -> key.scale != :Leaf, keys(graph_sim.temporal_state.caches)) + @test sim.outputs[(DomainModelKey(:churn_plant, :Plant, :domain_churn_leaf_controller), :created_count)] == [[1], [1], [2], [2]] + @test sim.outputs[(DomainModelKey(:churn_plant, :Plant, :domain_churn_leaf_controller), :removed_count)] == [[0], [1], [1], [2]] + @test sim.outputs[(DomainModelKey(:churn_plant, :Plant, :domain_churn_leaf_controller), :active_leaf_count)] == [[1], [0], [1], [0]] +end + +@testset "MTG-backed domain recursive subtree removal" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + internode = Node(plant, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + leaf = Node(internode, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 3)) + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + subtree_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainSubtreePruningModel()) |> + MultiScaleModel([ + :internode_flux => [:Internode], + :leaf_flux => [:Leaf], + ]) |> + InputBindings(; + internode_flux=(process=:domain_subtree_internode_flux, scale=:Internode, var=:internode_flux, policy=HoldLast()), + leaf_flux=(process=:domain_subtree_leaf_flux, scale=:Leaf, var=:leaf_flux, policy=HoldLast()), + ) |> + TimeStepModel(Dates.Hour(1)), + Status( + removed_count=0, + removed_internode_id=0, + removed_leaf_id=0, + remaining_internode_count=0, + remaining_leaf_count=0, + ), + ), + :Internode => ( + ModelSpec(DomainSubtreeInternodeFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + :Leaf => ( + ModelSpec(DomainSubtreeLeafFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + sim = run!( + scene, + SimulationMapping(Domain(:subtree_plant, subtree_mapping; kind=:plant, selector=plant)), + meteo, + check=true, + ) + + plant_status = only(status(sim, :subtree_plant, :Plant)) + graph_sim = only(sim.domain_states[:subtree_plant].simulations) + @test plant_status.removed_count == 1 + @test plant_status.removed_internode_id == node_id(internode) + @test plant_status.removed_leaf_id == node_id(leaf) + @test plant_status.remaining_internode_count == 0 + @test plant_status.remaining_leaf_count == 0 + @test length(plant_status.internode_flux) == 0 + @test length(plant_status.leaf_flux) == 0 + @test get(status(graph_sim), :Internode, Status[]) == Status[] + @test get(status(graph_sim), :Leaf, Status[]) == Status[] + @test status(sim, :subtree_plant, :Internode) == Status[] + @test status(sim, :subtree_plant, :Leaf) == Status[] + @test status(sim, :Internode) == Status[] + @test status(sim, :Leaf) == Status[] + @test only(row.nstatuses for row in explain_domain_statuses(sim) if row.domain == :subtree_plant && row.scale == :Internode) == 0 + @test only(row.nstatuses for row in explain_domain_statuses(sim) if row.domain == :subtree_plant && row.scale == :Leaf) == 0 + @test all(key -> key.node_id != plant_status.removed_internode_id, keys(graph_sim.temporal_state.streams)) + @test all(key -> key.node_id != plant_status.removed_leaf_id, keys(graph_sim.temporal_state.streams)) + @test all(key -> key.node_id != plant_status.removed_internode_id, keys(graph_sim.temporal_state.caches)) + @test all(key -> key.node_id != plant_status.removed_leaf_id, keys(graph_sim.temporal_state.caches)) + @test sim.outputs[(DomainModelKey(:subtree_plant, :Internode, :domain_subtree_internode_flux), :internode_flux)] == [[], []] + @test sim.outputs[(DomainModelKey(:subtree_plant, :Leaf, :domain_subtree_leaf_flux), :leaf_flux)] == [[], []] + @test sim.outputs[(DomainModelKey(:subtree_plant, :Plant, :domain_subtree_pruning), :remaining_internode_count)] == [[0], [0]] + @test sim.outputs[(DomainModelKey(:subtree_plant, :Plant, :domain_subtree_pruning), :remaining_leaf_count)] == [[0], [0]] +end + +@testset "MTG-backed domain topology reparenting" begin + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + internode_1 = Node(plant, MultiScaleTreeGraph.NodeMTG("/", :Internode, 1, 2)) + internode_2 = Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Internode, 2, 2)) + leaf = Node(internode_1, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 3)) + meteo = Weather([ + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, Ri_PAR_f=100.0, duration=Dates.Hour(1)) + for _ in 1:2 + ]) + + reparent_mapping = ModelMapping( + :Plant => ( + ModelSpec(DomainReparentLeafControllerModel()) |> + MultiScaleModel([:leaf_flux => [:Leaf]]) |> + InputBindings(; leaf_flux=(process=:domain_subtree_leaf_flux, scale=:Leaf, var=:leaf_flux, policy=HoldLast())) |> + TimeStepModel(Dates.Hour(1)), + Status(reparented_count=0, new_parent_id=0, leaf_parent_id=0, active_leaf_count=0), + ), + :Internode => ( + ModelSpec(DomainSubtreeInternodeFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + :Leaf => ( + ModelSpec(DomainSubtreeLeafFluxModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + + sim = run!( + scene, + SimulationMapping(Domain(:reparented_plant, reparent_mapping; kind=:plant, selector=plant)), + meteo, + check=true, + ) + + plant_status = only(status(sim, :reparented_plant, :Plant)) + graph_sim = only(sim.domain_states[:reparented_plant].simulations) + @test parent(leaf) === internode_2 + @test !any(node -> node === leaf, children(internode_1)) + @test count(node -> node === leaf, children(internode_2)) == 1 + @test plant_status.reparented_count == 1 + @test plant_status.new_parent_id == node_id(internode_2) + @test plant_status.leaf_parent_id == node_id(internode_2) + @test plant_status.active_leaf_count == 1 + @test length(plant_status.leaf_flux) == 1 + @test length(status(sim, :reparented_plant, :Internode)) == 2 + @test length(status(sim, :reparented_plant, :Leaf)) == 1 + @test all(key -> key.node_id != 0, keys(graph_sim.temporal_state.streams)) + @test sim.outputs[(DomainModelKey(:reparented_plant, :Leaf, :domain_subtree_leaf_flux), :leaf_flux)] == [[1.0], [1.0]] + @test sim.outputs[(DomainModelKey(:reparented_plant, :Plant, :domain_reparent_leaf_controller), :leaf_parent_id)] == [[node_id(internode_2)], [node_id(internode_2)]] + @test_throws ErrorException reparent_organ!(internode_2, leaf, graph_sim) +end diff --git a/test/test-environment-backends.jl b/test/test-environment-backends.jl new file mode 100644 index 000000000..2d04fa783 --- /dev/null +++ b/test/test-environment-backends.jl @@ -0,0 +1,375 @@ +using Dates + +PlantSimEngine.@process "environment_probe" verbose = false +PlantSimEngine.@process "environment_temperature_update" verbose = false +PlantSimEngine.@process "environment_bad_output" verbose = false +PlantSimEngine.@process "environment_graph_leaf_probe" verbose = false +PlantSimEngine.@process "environment_graph_temperature_update" verbose = false +PlantSimEngine.@process "environment_scene_hard_graph_update" verbose = false + +struct EnvironmentProbeModel <: AbstractEnvironment_ProbeModel end +struct EnvironmentTemperatureUpdateModel <: AbstractEnvironment_Temperature_UpdateModel end +struct EnvironmentBadOutputModel <: AbstractEnvironment_Bad_OutputModel end +struct EnvironmentGraphLeafProbeModel <: AbstractEnvironment_Graph_Leaf_ProbeModel end +struct EnvironmentGraphTemperatureUpdateModel <: AbstractEnvironment_Graph_Temperature_UpdateModel end +struct EnvironmentSceneHardGraphUpdateModel <: AbstractEnvironment_Scene_Hard_Graph_UpdateModel end + +PlantSimEngine.inputs_(::EnvironmentProbeModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentProbeModel) = (meteo_seen=0.0,) +PlantSimEngine.meteo_inputs_(::EnvironmentProbeModel) = (T=0.0, CO2=400.0) + +function PlantSimEngine.run!(::EnvironmentProbeModel, models, status, meteo, constants=nothing, extra=nothing) + status.meteo_seen = meteo.T + 0.001 * meteo.CO2 + return nothing +end + +PlantSimEngine.inputs_(::EnvironmentTemperatureUpdateModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentTemperatureUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_inputs_(::EnvironmentTemperatureUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_outputs_(::EnvironmentTemperatureUpdateModel) = (T=0.0,) + +function PlantSimEngine.run!(::EnvironmentTemperatureUpdateModel, models, status, meteo, constants=nothing, extra=nothing) + status.T = meteo.T + 1.0 + return nothing +end + +PlantSimEngine.inputs_(::EnvironmentBadOutputModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentBadOutputModel) = NamedTuple() +PlantSimEngine.meteo_inputs_(::EnvironmentBadOutputModel) = (T=0.0,) +PlantSimEngine.meteo_outputs_(::EnvironmentBadOutputModel) = (T=0.0,) + +function PlantSimEngine.run!(::EnvironmentBadOutputModel, models, status, meteo, constants=nothing, extra=nothing) + return nothing +end + +PlantSimEngine.inputs_(::EnvironmentGraphLeafProbeModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentGraphLeafProbeModel) = (meteo_seen=0.0,) +PlantSimEngine.meteo_inputs_(::EnvironmentGraphLeafProbeModel) = (T=0.0, CO2=400.0) + +function PlantSimEngine.run!(::EnvironmentGraphLeafProbeModel, models, status, meteo, constants=nothing, extra=nothing) + status.meteo_seen = meteo.T + 0.001 * meteo.CO2 + return nothing +end + +PlantSimEngine.inputs_(::EnvironmentGraphTemperatureUpdateModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentGraphTemperatureUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_inputs_(::EnvironmentGraphTemperatureUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_outputs_(::EnvironmentGraphTemperatureUpdateModel) = (T=0.0,) + +function PlantSimEngine.run!(::EnvironmentGraphTemperatureUpdateModel, models, status, meteo, constants=nothing, extra=nothing) + status.T = meteo.T + 2.0 + return nothing +end + +PlantSimEngine.dep(::EnvironmentSceneHardGraphUpdateModel) = ( + leaf_temperature=HardDomains(kind=:plant, scale=:Leaf, process=:environment_graph_temperature_update), +) +PlantSimEngine.inputs_(::EnvironmentSceneHardGraphUpdateModel) = NamedTuple() +PlantSimEngine.outputs_(::EnvironmentSceneHardGraphUpdateModel) = (hard_temperature_sum=0.0,) + +function PlantSimEngine.run!(::EnvironmentSceneHardGraphUpdateModel, models, status, meteo, constants=nothing, extra=nothing) + targets = dependency_targets(extra, :leaf_temperature) + for target in targets + run_target!(target; publish=true) + end + status.hard_temperature_sum = sum(target.status.T for target in targets) + return nothing +end + +struct ProbeEnvironmentBackend <: AbstractEnvironmentBackend + nsteps::Int + base_seconds::Float64 +end + +PlantSimEngine.get_nsteps(backend::ProbeEnvironmentBackend) = backend.nsteps +PlantSimEngine.base_step_seconds(backend::ProbeEnvironmentBackend) = backend.base_seconds +PlantSimEngine.environment_variables(::ProbeEnvironmentBackend) = Set([:T, :CO2, :Ca]) + +function PlantSimEngine.sample( + ::ProbeEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + time +) + variable == :CO2 && return 410.0 + variable == :Ca && return 420.0 + variable == :T || error("Unexpected variable `$(variable)`.") + offset = support.domain == :plant_b ? 100.0 : 0.0 + return offset + 10.0 + float(time) +end + +struct MissingCO2EnvironmentBackend <: AbstractEnvironmentBackend end + +PlantSimEngine.get_nsteps(::MissingCO2EnvironmentBackend) = 1 +PlantSimEngine.base_step_seconds(::MissingCO2EnvironmentBackend) = 3600.0 +PlantSimEngine.environment_variables(::MissingCO2EnvironmentBackend) = Set([:T]) +PlantSimEngine.sample(::MissingCO2EnvironmentBackend, variable::Symbol, support::EnvironmentSupport, time) = 20.0 + +mutable struct ScatteringEnvironmentBackend <: AbstractEnvironmentBackend + nsteps::Int + base_seconds::Float64 + writes::Vector{NamedTuple} + index_updates::Vector{Any} +end + +PlantSimEngine.get_nsteps(backend::ScatteringEnvironmentBackend) = backend.nsteps +PlantSimEngine.base_step_seconds(backend::ScatteringEnvironmentBackend) = backend.base_seconds +PlantSimEngine.environment_variables(::ScatteringEnvironmentBackend) = Set([:T]) +PlantSimEngine.sample(::ScatteringEnvironmentBackend, variable::Symbol, support::EnvironmentSupport, time) = 20.0 + float(time) + +function PlantSimEngine.scatter!( + backend::ScatteringEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + value, + time +) + push!(backend.writes, (domain=support.domain, process=support.process, variable=variable, value=value, time=time)) + return nothing +end + +function PlantSimEngine.update_index!(backend::ScatteringEnvironmentBackend, entities) + push!( + backend.index_updates, + [ + (domain=entity.domain, kind=entity.kind, scale=entity.scale, nstatuses=length(entity.statuses)) + for entity in entities + ], + ) + return nothing +end + +mutable struct GraphEnvironmentBackend <: AbstractEnvironmentBackend + nsteps::Int + base_seconds::Float64 + writes::Vector{NamedTuple} + index_updates::Vector{Any} +end + +PlantSimEngine.get_nsteps(backend::GraphEnvironmentBackend) = backend.nsteps +PlantSimEngine.base_step_seconds(backend::GraphEnvironmentBackend) = backend.base_seconds +PlantSimEngine.environment_variables(::GraphEnvironmentBackend) = Set([:T, :CO2]) + +function PlantSimEngine.sample( + ::GraphEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + time +) + variable == :CO2 && return 410.0 + variable == :T || error("Unexpected variable `$(variable)`.") + return 20.0 + float(time) + 0.1 * node_id(support.status.node) +end + +function PlantSimEngine.scatter!( + backend::GraphEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + value, + time +) + push!( + backend.writes, + ( + domain=support.domain, + scale=support.scale, + process=support.process, + node_id=node_id(support.status.node), + variable=variable, + value=value, + time=time, + ), + ) + return nothing +end + +function PlantSimEngine.update_index!(backend::GraphEnvironmentBackend, entities) + push!( + backend.index_updates, + [ + (domain=entity.domain, kind=entity.kind, scale=entity.scale, nstatuses=length(entity.statuses)) + for entity in entities + ], + ) + return nothing +end + +@testset "Environment backends" begin + support = EnvironmentSupport(:plant_a, :Default, :environment_probe, nothing) + global_backend = GlobalConstant(Atmosphere(T=20.0, Rh=0.65, Wind=1.0, CO2=410.0, duration=Dates.Hour(1))) + @test sample(global_backend, :T, support, 1.0) == 20.0 + @test PlantSimEngine.get_nsteps(global_backend) == 1 + @test base_step_seconds(global_backend) == 3600.0 + @test environment_variables(GlobalConstant(nothing)) == Set{Symbol}() + + mapping = ModelMapping( + ModelSpec(EnvironmentProbeModel()) |> TimeStepModel(Dates.Hour(1)), + status=(meteo_seen=0.0,), + ) + simulation_mapping = SimulationMapping( + Domain(:plant_a, mapping; kind=:plant), + Domain(:plant_b, mapping; kind=:plant), + ) + + sim = run!(simulation_mapping, ProbeEnvironmentBackend(3, 3600.0), check=true) + @test status(sim, :plant_a).meteo_seen ≈ 13.41 + @test status(sim, :plant_b).meteo_seen ≈ 113.41 + + environment = explain_environment(sim) + @test environment.backend == ProbeEnvironmentBackend + @test environment.nsteps == 3 + @test environment.base_step_seconds == 3600.0 + @test :CO2 in environment.variables + + bound_mapping = ModelMapping( + ModelSpec(EnvironmentProbeModel()) |> + TimeStepModel(Dates.Hour(1)) |> + MeteoBindings(; CO2=(source=:Ca, reducer=MeanReducer())), + status=(meteo_seen=0.0,), + ) + bound_sim = run!( + SimulationMapping(Domain(:plant_a, bound_mapping; kind=:plant)), + ProbeEnvironmentBackend(1, 3600.0), + check=true, + ) + @test status(bound_sim, :plant_a).meteo_seen ≈ 11.42 + + @test_throws "CO2" run!( + SimulationMapping(Domain(:plant_a, mapping; kind=:plant)), + MissingCO2EnvironmentBackend(), + check=true, + ) + @test_throws "CO2" run!( + SimulationMapping(Domain(:plant_a, mapping; kind=:plant)), + nothing, + check=true, + ) + + update_mapping = ModelMapping( + ModelSpec(EnvironmentTemperatureUpdateModel()) |> TimeStepModel(Dates.Hour(1)), + status=(T=0.0,), + ) + scattering_backend = ScatteringEnvironmentBackend(2, 3600.0, NamedTuple[], Any[]) + scatter_sim = run!( + SimulationMapping(Domain(:plant_a, update_mapping; kind=:plant)), + scattering_backend, + check=true, + ) + @test status(scatter_sim, :plant_a).T == 23.0 + @test scattering_backend.writes == [ + (domain=:plant_a, process=:environment_temperature_update, variable=:T, value=22.0, time=1.0), + (domain=:plant_a, process=:environment_temperature_update, variable=:T, value=23.0, time=2.0), + ] + @test scattering_backend.index_updates == [ + [(domain=:plant_a, kind=:plant, scale=:Default, nstatuses=1)], + [(domain=:plant_a, kind=:plant, scale=:Default, nstatuses=1)], + ] + + @test_throws "GlobalConstant is immutable" run!( + SimulationMapping(Domain(:plant_a, update_mapping; kind=:plant)), + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, duration=Dates.Hour(1)), + check=true, + ) + + bad_mapping = ModelMapping( + ModelSpec(EnvironmentBadOutputModel()) |> TimeStepModel(Dates.Hour(1)), + status=NamedTuple(), + ) + @test_throws "status does not contain" run!( + SimulationMapping(Domain(:plant_a, bad_mapping; kind=:plant)), + ScatteringEnvironmentBackend(1, 3600.0, NamedTuple[], Any[]), + check=true, + ) + + scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + plant = Node(scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + leaf_1 = Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + leaf_2 = Node(plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 2, 2)) + leaf_ids = sort([node_id(leaf_1), node_id(leaf_2)]) + graph_mapping = ModelMapping( + :Leaf => ( + ModelSpec(EnvironmentGraphLeafProbeModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + graph_backend = GraphEnvironmentBackend(2, 3600.0, NamedTuple[], Any[]) + graph_sim = run!( + scene, + SimulationMapping(Domain(:plant_a, graph_mapping; kind=:plant, selector=plant)), + graph_backend, + check=true, + ) + graph_values = graph_sim.outputs[(DomainModelKey(:plant_a, :Leaf, :environment_graph_leaf_probe), :meteo_seen)] + @test graph_values == [ + [20.0 + 1.0 + 0.1 * leaf_ids[1] + 0.410, 20.0 + 1.0 + 0.1 * leaf_ids[2] + 0.410], + [20.0 + 2.0 + 0.1 * leaf_ids[1] + 0.410, 20.0 + 2.0 + 0.1 * leaf_ids[2] + 0.410], + ] + @test graph_backend.index_updates == [ + [(domain=:plant_a, kind=:plant, scale=:Leaf, nstatuses=2)], + [(domain=:plant_a, kind=:plant, scale=:Leaf, nstatuses=2)], + ] + + scatter_scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + scatter_plant = Node(scatter_scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + scatter_leaf = Node(scatter_plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + scatter_mapping = ModelMapping( + :Leaf => ( + ModelSpec(EnvironmentGraphTemperatureUpdateModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + graph_scatter_backend = GraphEnvironmentBackend(1, 3600.0, NamedTuple[], Any[]) + graph_scatter_sim = run!( + scatter_scene, + SimulationMapping(Domain(:plant_a, scatter_mapping; kind=:plant, selector=scatter_plant)), + graph_scatter_backend, + check=true, + ) + @test only(status(graph_scatter_sim, :plant_a, :Leaf)).T ≈ 20.0 + 1.0 + 0.1 * node_id(scatter_leaf) + 2.0 + @test graph_scatter_backend.writes == [ + ( + domain=:plant_a, + scale=:Leaf, + process=:environment_graph_temperature_update, + node_id=node_id(scatter_leaf), + variable=:T, + value=20.0 + 1.0 + 0.1 * node_id(scatter_leaf) + 2.0, + time=1.0, + ), + ] + + hard_scatter_scene = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + hard_scatter_plant = Node(hard_scatter_scene, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + hard_scatter_leaf = Node(hard_scatter_plant, MultiScaleTreeGraph.NodeMTG("+", :Leaf, 1, 2)) + hard_scatter_mapping = ModelMapping( + :Leaf => ( + ModelSpec(EnvironmentGraphTemperatureUpdateModel()) |> TimeStepModel(Dates.Hour(1)), + ), + ) + hard_scatter_scene_mapping = ModelMapping( + ModelSpec(EnvironmentSceneHardGraphUpdateModel()) |> TimeStepModel(Dates.Hour(1)), + status=(hard_temperature_sum=0.0,), + ) + hard_graph_scatter_backend = GraphEnvironmentBackend(1, 3600.0, NamedTuple[], Any[]) + hard_graph_scatter_sim = run!( + hard_scatter_scene, + SimulationMapping( + Domain(:plant_a, hard_scatter_mapping; kind=:plant, selector=hard_scatter_plant), + Domain(:scene, hard_scatter_scene_mapping; kind=:scene), + ), + hard_graph_scatter_backend, + check=true, + ) + expected_hard_T = 20.0 + 1.0 + 0.1 * node_id(hard_scatter_leaf) + 2.0 + @test only(status(hard_graph_scatter_sim, :plant_a, :Leaf)).T ≈ expected_hard_T + @test status(hard_graph_scatter_sim, :scene).hard_temperature_sum ≈ expected_hard_T + @test hard_graph_scatter_backend.writes == [ + ( + domain=:plant_a, + scale=:Leaf, + process=:environment_graph_temperature_update, + node_id=node_id(hard_scatter_leaf), + variable=:T, + value=expected_hard_T, + time=1.0, + ), + ] +end diff --git a/test/test-maespa-domain-example.jl b/test/test-maespa-domain-example.jl new file mode 100644 index 000000000..e4c741d65 --- /dev/null +++ b/test/test-maespa-domain-example.jl @@ -0,0 +1,133 @@ +include("../examples/maespa_domain_example.jl") + +@testset "MAESPA-style domain example" begin + result = run_maespa_example(; nhours=24, check=true) + sim = result.simulation + + @test length(status(sim, :Leaf)) == 5 + @test length(status(sim, :plant_A, :Leaf)) == 2 + @test length(status(sim, :plant_B, :Leaf)) == 3 + + scene_status = status(sim, :scene) + last_meteo = last(maespa_meteo(; nhours=24)) + vpd_above = max(0.01, last_meteo.VPD) + @test isfinite(scene_status.canopy_tair) + @test isfinite(scene_status.canopy_vpd) + @test isfinite(scene_status.canopy_rh) + @test isfinite(scene_status.canopy_htot) + @test isfinite(scene_status.canopy_gcanop) + @test scene_status.canopy_tair >= last_meteo.T - 10.0 + @test scene_status.canopy_tair <= last_meteo.T + 10.0 + @test scene_status.canopy_vpd >= max(0.01, vpd_above - 1.5) + @test scene_status.canopy_vpd <= vpd_above + 1.5 + @test scene_status.canopy_vpd > 0.0 + @test 0.0 <= scene_status.canopy_rh <= 1.0 + @test isfinite(scene_status.scene_transpiration) + @test scene_status.scene_transpiration > 0.0 + @test scene_status.iterations > 0 + @test scene_status.leaf_area ≈ sum(st.leaf_area for st in status(sim, :Leaf)) + @test scene_status.lai ≈ scene_status.leaf_area + @test scene_status.leaf_areas ≈ getproperty.(status(sim, :Leaf), :leaf_area) + + leaf_statuses = status(sim, :Leaf) + @test all(st -> isfinite(st.Tₗ), leaf_statuses) + @test all(st -> isfinite(st.A), leaf_statuses) + @test all(st -> isfinite(st.λE), leaf_statuses) + @test any(st -> abs(st.Tₗ - scene_status.canopy_tair) > 1.0e-6, leaf_statuses) + + plant_a_status = only(status(sim, :plant_A, :Plant)) + plant_b_status = only(status(sim, :plant_B, :Plant)) + @test plant_a_status.daily_growth > 0.0 + @test plant_b_status.daily_growth > 0.0 + @test plant_a_status.daily_growth != plant_b_status.daily_growth + @test plant_a_status.leaf_pool != plant_b_status.leaf_pool + + deps = explain_domain_dependencies(sim) + @test count(row -> row.mode == :hard_domain && row.dependency == :energy_balance, deps) == 2 + @test count(row -> row.mode == :hard_domain && row.dependency == :soil, deps) == 1 + + @test length(sim.outputs[(DomainModelKey(:plant_A, :Leaf, :energy_balance), :λE)]) == 2 * 24 + @test length(sim.outputs[(DomainModelKey(:plant_B, :Leaf, :energy_balance), :λE)]) == 3 * 24 + @test length(sim.outputs[(DomainModelKey(:soil, :Default, :soil_water), :psi_soil)]) == 24 + @test length(sim.outputs[(DomainModelKey(:scene, :Default, :lai_dynamic), :lai)]) == 1 + @test length(sim.outputs[(DomainModelKey(:scene, :Default, :scene_eb), :scene_transpiration)]) == 24 + @test status(sim, :soil).transpiration ≈ scene_status.scene_transpiration + @test status(sim, :soil).psi_soil ≈ scene_status.psi_soil +end + +@testset "MAESPA-style domain example validation" begin + mtg = build_maespa_scene() + meteo = maespa_meteo(; nhours=1) + + soil_mapping = ModelMapping( + ModelSpec(SoilWater(0.45, -0.03, 4.4, 0.25, 0.75)) |> TimeStepModel(Dates.Hour(1)), + status=(theta1=0.33, theta2=0.36, psi_soil=-0.10, transpiration=0.0, infiltration=0.0), + ) + scene_mapping = ModelMapping( + ModelSpec(LAIModel(1.0)) |> TimeStepModel(Dates.Hour(1)), + ModelSpec(SceneEB(25, 0.03, 0.005)) |> + Calls( + :energy_balance => Many(kind=:plant, scale=:Leaf, process=:energy_balance), + :soil => One(kind=:soil, process=:soil_water), + ) |> + TimeStep(Dates.Hour(1)), + status=( + leaf_area=0.0, + lai=0.0, + canopy_tair=20.0, + canopy_vpd=1.0, + canopy_rh=0.7, + canopy_htot=0.0, + canopy_gcanop=0.0, + scene_transpiration=0.0, + scene_assimilation=0.0, + psi_soil=-0.1, + iterations=0, + ), + ) + missing_leaf_mapping = SimulationMapping( + Domain(:soil, soil_mapping; kind=:soil), + Domain(:scene, scene_mapping; kind=:scene), + ) + @test_throws "Hard domain dependency `energy_balance`" run!(mtg, missing_leaf_mapping, meteo, check=true, executor=SequentialEx()) + + soil_target = (status=Status(psi_soil=-0.1),) + scene_status = Status(lai=0.0, leaf_area=0.0) + @test_throws "SceneEB did not converge after 0 iterations" _solve_scene_energy_balance!( + SceneEB(0, 0.03, 0.005), + ModelTarget[], + soil_target, + scene_status, + first(meteo), + ) +end + +@testset "MAESPA-style canopy helper functions" begin + lai_model = LAIModel(2.0) + lai_status = Status(leaf_areas=[0.5, 1.0], leaf_area=0.0, lai=0.0) + PlantSimEngine.run!(lai_model, nothing, lai_status, nothing, nothing) + @test lai_status.leaf_area ≈ 1.5 + @test lai_status.lai ≈ 0.75 + + m = SceneEB(25, 0.03, 0.005; ground_area=2.0) + + low_wind = gbcanms(0.0, m.zht, m.tree_height; gbcan_min=m.gbcan_min, von_karman=m.von_karman) + @test low_wind.canopy_air_ms ≈ m.gbcan_min + @test isfinite(low_wind.soil_canopy_ms) + + meteo_above = Atmosphere(T=25.0, Rh=0.50, Wind=1.2, Ri_PAR_f=800.0, Ri_SW_f=400.0, duration=Dates.Hour(1)) + canopy_meteo = Atmosphere(T=25.0, Rh=0.50, Wind=1.2, P=meteo_above.P, Ri_PAR_f=800.0, Ri_SW_f=400.0, duration=Dates.Hour(1)) + hot_fluxes = (rn=5000.0, lambda_e=-5000.0, a=0.0, lai=0.75, rad_interc=0.0) + hot = tvpdcanopcalc(m, hot_fluxes, meteo_above, canopy_meteo, PlantMeteo.Constants()) + @test hot.tair <= meteo_above.T + 10.0 + @test hot.vpd <= max(0.01, meteo_above.VPD) + 1.5 + + wet_fluxes = (rn=-5000.0, lambda_e=5000.0, a=0.0, lai=0.75, rad_interc=0.0) + wet = tvpdcanopcalc(m, wet_fluxes, meteo_above, canopy_meteo, PlantMeteo.Constants()) + @test wet.tair >= meteo_above.T - 10.0 + @test wet.vpd >= max(0.01, meteo_above.VPD - 1.5) + @test wet.vpd >= 0.01 + @test 0.0 <= wet.rh <= 1.0 + @test meteo_above.T == 25.0 + @test max(0.01, meteo_above.VPD) == meteo_above.VPD +end diff --git a/test/test-meteo-traits.jl b/test/test-meteo-traits.jl new file mode 100644 index 000000000..1e983ec21 --- /dev/null +++ b/test/test-meteo-traits.jl @@ -0,0 +1,60 @@ +using Dates + +PlantSimEngine.@process "meteo_trait_consumer" verbose = false + +struct MeteoTraitConsumerModel <: AbstractMeteo_Trait_ConsumerModel end + +PlantSimEngine.inputs_(::MeteoTraitConsumerModel) = NamedTuple() +PlantSimEngine.outputs_(::MeteoTraitConsumerModel) = (meteo_seen=0.0,) +PlantSimEngine.meteo_inputs_(::MeteoTraitConsumerModel) = (T=0.0, CO2=400.0) + +function PlantSimEngine.run!(::MeteoTraitConsumerModel, models, status, meteo, constants=nothing, extra=nothing) + status.meteo_seen = meteo.T + return nothing +end + +@testset "Meteo traits" begin + specs = Dict(:Leaf => Dict(:meteo_trait_consumer => ModelSpec(MeteoTraitConsumerModel()))) + + @test_throws "CO2" PlantSimEngine.validate_meteo_inputs( + specs, + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, duration=Dates.Hour(1)) + ) + + @test PlantSimEngine.validate_meteo_inputs( + specs, + (T=20.0, CO2=410.0, duration=Dates.Hour(1)) + ) === nothing + + bound_spec = + ModelSpec(MeteoTraitConsumerModel()) |> + MeteoBindings(; CO2=(source=:Ca, reducer=MeanReducer())) + bound_specs = Dict(:Leaf => Dict(:meteo_trait_consumer => bound_spec)) + + @test_throws "Ca" PlantSimEngine.validate_meteo_inputs( + bound_specs, + (T=20.0, CO2=410.0, duration=Dates.Hour(1)) + ) + @test PlantSimEngine.validate_meteo_inputs( + bound_specs, + (T=20.0, Ca=410.0, duration=Dates.Hour(1)) + ) === nothing + + mapping = ModelMapping( + MeteoTraitConsumerModel(), + status=(meteo_seen=0.0,), + ) + @test_throws "CO2" PlantSimEngine.validate_meteo_inputs( + mapping, + Atmosphere(T=20.0, Rh=0.65, Wind=1.0, duration=Dates.Hour(1)) + ) + @test PlantSimEngine.validate_meteo_inputs( + mapping, + (T=20.0, CO2=410.0, duration=Dates.Hour(1)) + ) === nothing + + @test PlantSimEngine.validate_meteo_inputs( + Dict(:Leaf => (MeteoTraitConsumerModel(),)), + (T=20.0, CO2=410.0, duration=Dates.Hour(1)) + ) === nothing +end diff --git a/test/test-mtg-dynamic.jl b/test/test-mtg-dynamic.jl index 28d7ef2d9..efcfa9962 100644 --- a/test/test-mtg-dynamic.jl +++ b/test/test-mtg-dynamic.jl @@ -79,6 +79,12 @@ out = run!(sim,meteo) @test length(mtg) == 9 @test length(st[:Scene]) == length(st[:Soil]) == length(st[:Plant]) == 1 @test length(st[:Internode]) == length(st[:Leaf]) == 3 + @test node_id.(getproperty.(st[:Leaf], :node)) == sort(node_id.(getproperty.(st[:Leaf], :node))) + leaf_assimilation_refs = [PlantSimEngine.refvalue(leaf_status, :carbon_assimilation) for leaf_status in st[:Leaf]] + @test all( + ref_pair -> first(ref_pair) === last(ref_pair), + zip(parent(sim.status_templates[:Plant][:carbon_assimilation]), leaf_assimilation_refs), + ) @test st[:Internode][1].TT_cu_emergence == 0.0 @test st[:Internode][end].TT_cu_emergence == 25.0 diff --git a/test/test-mtg-multiscale-cyclic-dep.jl b/test/test-mtg-multiscale-cyclic-dep.jl index c9f9a4c88..1f92c6311 100644 --- a/test/test-mtg-multiscale-cyclic-dep.jl +++ b/test/test-mtg-multiscale-cyclic-dep.jl @@ -114,7 +114,7 @@ end #out = @test_nowarn run!(mtg, mapping_nocyclic, meteo, tracked_outputs=out_vars, executor=SequentialEx()) nsteps = PlantSimEngine.get_nsteps(meteo) - sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) + sim = PlantSimEngine.GraphSimulation(mtg, mapping_nocyclic, nsteps=nsteps, check=true, outputs=out_vars) out = @test_nowarn run!(sim,meteo) st = status(sim) diff --git a/test/test-multirate-output-export.jl b/test/test-multirate-output-export.jl index 30162d09d..0c373518f 100644 --- a/test/test-multirate-output-export.jl +++ b/test/test-multirate-output-export.jl @@ -44,6 +44,50 @@ function PlantSimEngine.run!(::MRDefaultSceneAggModel, models, status, meteo, co end PlantSimEngine.timespec(::Type{<:MRDefaultSceneAggModel}) = ClockSpec(4.0, 1.0) +PlantSimEngine.@process "mrdynamicgrow" verbose = false +struct MRDynamicGrowModel <: AbstractMrdynamicgrowModel end +PlantSimEngine.inputs_(::MRDynamicGrowModel) = NamedTuple() +PlantSimEngine.outputs_(::MRDynamicGrowModel) = (nleaves=0.0,) +function PlantSimEngine.run!(::MRDynamicGrowModel, models, status, meteo, constants=nothing, extra=nothing) + if length(PlantSimEngine.status(extra)[:Leaf]) == 0 + add_organ!(status.node, extra, "+", :Leaf, 2; check=true) + end + status.nleaves = length(PlantSimEngine.status(extra)[:Leaf]) + return nothing +end + +PlantSimEngine.@process "mrdynamicgrowmany" verbose = false +struct MRDynamicGrowManyModel <: AbstractMrdynamicgrowmanyModel end +PlantSimEngine.inputs_(::MRDynamicGrowManyModel) = NamedTuple() +PlantSimEngine.outputs_(::MRDynamicGrowManyModel) = (nleaves=0.0,) +function PlantSimEngine.run!(::MRDynamicGrowManyModel, models, status, meteo, constants=nothing, extra=nothing) + while length(PlantSimEngine.status(extra)[:Leaf]) < 2 + add_organ!(status.node, extra, "+", :Leaf, 2; check=true) + end + status.nleaves = length(PlantSimEngine.status(extra)[:Leaf]) + return nothing +end + +PlantSimEngine.@process "mrdynamicleafsource" verbose = false +struct MRDynamicLeafSourceModel <: AbstractMrdynamicleafsourceModel end +PlantSimEngine.inputs_(::MRDynamicLeafSourceModel) = (nleaves=0.0,) +PlantSimEngine.outputs_(::MRDynamicLeafSourceModel) = (X=-Inf,) +function PlantSimEngine.run!(::MRDynamicLeafSourceModel, models, status, meteo, constants=nothing, extra=nothing) + status.X = status.nleaves * meteo.T / 10.0 + return nothing +end + +function _mr_custom_plant_scope(node, scale, process) + current = node + while !isnothing(current) + if MultiScaleTreeGraph.symbol(current) == :Plant + return ScopeId(:custom_plant, MultiScaleTreeGraph.node_id(current)) + end + current = parent(current) + end + return ScopeId(:custom_plant, 0) +end + @testset "Multi-rate output export API" begin mtg = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) plant = Node(mtg, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) @@ -130,7 +174,7 @@ PlantSimEngine.timespec(::Type{<:MRDefaultSceneAggModel}) = ClockSpec(4.0, 1.0) # Optional direct export return from run! on MTG + mapping entry point. out_status_mtg, out_requested_mtg = run!( mtg, - Dict( + ModelMapping( :Leaf => ( ModelSpec(MRExportSourceModel(Ref(0))) |> TimeStepModel(1.0), @@ -143,6 +187,92 @@ PlantSimEngine.timespec(::Type{<:MRDefaultSceneAggModel}) = ClockSpec(4.0, 1.0) ) @test haskey(out_status_mtg, :Leaf) @test out_requested_mtg[:x_mtg][:, :value] == [1.0, 2.0, 3.0, 4.0] + + # Dynamic statuses created after GraphSimulation initialization are visible + # to online OutputRequest exports. + dynamic_mtg = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + dynamic_plant = Node(dynamic_mtg, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + dynamic_meteo = Weather([ + Atmosphere(T=10.0, Wind=1.0, Rh=0.65), + Atmosphere(T=20.0, Wind=1.0, Rh=0.65), + Atmosphere(T=30.0, Wind=1.0, Rh=0.65), + Atmosphere(T=40.0, Wind=1.0, Rh=0.65), + ]) + dynamic_mapping = ModelMapping( + :Plant => ( + ModelSpec(MRDynamicGrowModel()) |> + TimeStepModel(1.0) |> + ScopeModel(:plant), + ), + :Leaf => ( + ModelSpec(MRDynamicLeafSourceModel()) |> + MultiScaleModel([:nleaves => (:Plant => :nleaves)]) |> + TimeStepModel(1.0) |> + ScopeModel(:plant), + ), + ) + dynamic_sim = PlantSimEngine.GraphSimulation( + dynamic_mtg, + dynamic_mapping, + nsteps=4, + check=true, + outputs=Dict(:Leaf => (:X,)), + ) + run!( + dynamic_sim, + dynamic_meteo, + executor=SequentialEx(), + tracked_outputs=[ + OutputRequest(:Leaf, :X; name=:dynamic_hold, process=:mrdynamicleafsource, policy=HoldLast()), + OutputRequest(:Leaf, :X; name=:dynamic_sum2, process=:mrdynamicleafsource, policy=Integrate(), clock=ClockSpec(2.0, 1.0)), + ], + ) + dynamic_exported = collect_outputs(dynamic_sim; sink=DataFrame) + @test length(status(dynamic_sim)[:Leaf]) == 1 + @test dynamic_exported[:dynamic_hold][:, :timestep] == [1, 2, 3, 4] + @test dynamic_exported[:dynamic_hold][:, :value] == [1.0, 2.0, 3.0, 4.0] + @test dynamic_exported[:dynamic_sum2][:, :timestep] == [1, 3] + @test dynamic_exported[:dynamic_sum2][:, :value] == [1.0, 5.0] + + # Several dynamic statuses and callable scopes are resolved from live status + # vectors when temporal outputs are exported. + many_mtg = Node(MultiScaleTreeGraph.NodeMTG("/", :Scene, 1, 0)) + many_plant = Node(many_mtg, MultiScaleTreeGraph.NodeMTG("+", :Plant, 1, 1)) + many_mapping = ModelMapping( + :Plant => ( + ModelSpec(MRDynamicGrowManyModel()) |> + TimeStepModel(1.0) |> + ScopeModel(_mr_custom_plant_scope), + ), + :Leaf => ( + ModelSpec(MRDynamicLeafSourceModel()) |> + MultiScaleModel([:nleaves => (:Plant => :nleaves)]) |> + TimeStepModel(1.0) |> + ScopeModel(_mr_custom_plant_scope), + ), + ) + many_sim = PlantSimEngine.GraphSimulation( + many_mtg, + many_mapping, + nsteps=4, + check=true, + outputs=Dict(:Leaf => (:X,)), + ) + run!( + many_sim, + dynamic_meteo, + executor=SequentialEx(), + tracked_outputs=[ + OutputRequest(:Leaf, :X; name=:many_hold, process=:mrdynamicleafsource, policy=HoldLast()), + OutputRequest(:Leaf, :X; name=:many_sum2, process=:mrdynamicleafsource, policy=Integrate(), clock=ClockSpec(2.0, 1.0)), + ], + ) + many_exported = collect_outputs(many_sim; sink=DataFrame) + @test length(status(many_sim)[:Leaf]) == 2 + @test many_exported[:many_hold][:, :timestep] == [1, 1, 2, 2, 3, 3, 4, 4] + @test many_exported[:many_hold][:, :value] == [2.0, 2.0, 4.0, 4.0, 6.0, 6.0, 8.0, 8.0] + @test many_exported[:many_sum2][:, :timestep] == [1, 1, 3, 3] + @test many_exported[:many_sum2][:, :value] == [2.0, 2.0, 10.0, 10.0] end @testset "Multi-rate output export defaults on multi-scale mapping with timespec traits" begin diff --git a/test/test-multirate-runtime.jl b/test/test-multirate-runtime.jl index 2d88c7fd2..0a6004df6 100644 --- a/test/test-multirate-runtime.jl +++ b/test/test-multirate-runtime.jl @@ -281,7 +281,7 @@ PlantSimEngine.dep(::MRHardParentModel) = (mrhardchild=AbstractMrhardchildModel, PlantSimEngine.inputs_(::MRHardParentModel) = NamedTuple() PlantSimEngine.outputs_(::MRHardParentModel) = (A=-Inf,) function PlantSimEngine.run!(::MRHardParentModel, models, status, meteo, constants=nothing, extra=nothing) - run!(models.mrhardchild, models, status, meteo, constants, extra) + run_target!(models, status, :mrhardchild; meteo=meteo, constants=constants, extra=extra) status.A = 5.0 end @@ -435,15 +435,35 @@ PlantSimEngine.meteo_hint(::Type{<:MRMeteoHintConsumerModel}) = ( @test sim_ok.temporal_state.caches[key_dir].v == 10.0 @test sim_ok.temporal_state.caches[key_auto].v == 10.0 + mapping_renamed_auto = ModelMapping( + :Leaf => MRSourceModel(), + :Plant => ( + ModelSpec(MRConsumerModel()) |> + MultiScaleModel([:C => (:Leaf => :S)]) |> + TimeStepModel(ClockSpec(1.0, 0.0)), + ), + ) + sim_renamed_auto = PlantSimEngine.GraphSimulation(mtg, mapping_renamed_auto, nsteps=1, check=true) + consumer_node = only(filter( + n -> n.scale == :Plant && n.process == :mrconsumer, + PlantSimEngine.traverse_dependency_graph(dep(sim_renamed_auto), false), + )) + @test PlantSimEngine._candidate_producers(consumer_node, :C) == [(:mrsource, :S)] + mapping_conflict = ModelMapping( :Leaf => ( ModelSpec(MRConflict1Model()) |> TimeStepModel(1.0), ModelSpec(MRConflict2Model()) |> TimeStepModel(1.0), ), ) - sim_conflict = PlantSimEngine.GraphSimulation(mtg, mapping_conflict, nsteps=1, check=true, outputs=Dict(:Leaf => (:Z,))) - # Expectation 5: two canonical publishers of the same output are rejected. - @test_throws "Ambiguous canonical publishers" run!(sim_conflict, meteo, executor=SequentialEx()) + # Expectation 5: two canonical writers of the same output are rejected at graph construction. + @test_throws "Ambiguous canonical writers" PlantSimEngine.GraphSimulation( + mtg, + mapping_conflict, + nsteps=1, + check=true, + outputs=Dict(:Leaf => (:Z,)) + ) # Expectation 6: models run at different clocks; slower model holds last value between runs. source_counter = Ref(0) @@ -1250,7 +1270,7 @@ PlantSimEngine.meteo_hint(::Type{<:MRMeteoHintConsumerModel}) = ( @test input_bindings(spec_hard_same_rate).A.process == :mrhardparent @test status(sim_hard_same_rate)[:Leaf][1].B == 5.0 - # Expectation 24f: different-rate hard dependencies remain strict and require explicit disambiguation. + # Expectation 24f: hard dependency children cannot be scheduled independently from their parent. mapping_hard_different_rate = ModelMapping( :Leaf => ( ModelSpec(MRHardParentModel()) |> TimeStepModel(1.0), @@ -1258,7 +1278,7 @@ PlantSimEngine.meteo_hint(::Type{<:MRMeteoHintConsumerModel}) = ( ModelSpec(MRHardConsumerModel()) |> TimeStepModel(1.0), ), ) - @test_throws "Ambiguous inferred producer for input `A`" PlantSimEngine.GraphSimulation( + @test_throws "Hard dependency `mrhardchild`" PlantSimEngine.GraphSimulation( mtg, mapping_hard_different_rate, nsteps=1, @@ -1266,24 +1286,6 @@ PlantSimEngine.meteo_hint(::Type{<:MRMeteoHintConsumerModel}) = ( outputs=Dict(:Leaf => (:A, :B)) ) - mapping_hard_different_rate_explicit = ModelMapping( - :Leaf => ( - ModelSpec(MRHardParentModel()) |> TimeStepModel(1.0), - ModelSpec(MRHardChildModel()) |> TimeStepModel(2.0), - ModelSpec(MRHardConsumerModel()) |> - TimeStepModel(1.0) |> - InputBindings(; A=(process=:mrhardparent, var=:A)), - ), - ) - sim_hard_different_rate_explicit = PlantSimEngine.GraphSimulation( - mtg, - mapping_hard_different_rate_explicit, - nsteps=1, - check=true, - outputs=Dict(:Leaf => (:A, :B)) - ) - @test_throws "Ambiguous canonical publishers" run!(sim_hard_different_rate_explicit, meteo, executor=SequentialEx()) - # Expectation 25: missing producer remains allowed; model can rely on initialized/forced inputs. mapping_missing_input = ModelMapping( :Leaf => ( diff --git a/test/test-multirate-scaffolding.jl b/test/test-multirate-scaffolding.jl index 1312cd3cd..f34bdcd45 100644 --- a/test/test-multirate-scaffolding.jl +++ b/test/test-multirate-scaffolding.jl @@ -1,5 +1,6 @@ using PlantSimEngine using PlantSimEngine.Examples +using MultiScaleTreeGraph using Test @testset "Multi-rate scaffolding" begin @@ -12,9 +13,24 @@ using Test @test output_policy(m) == NamedTuple() @test isnothing(timestep_hint(m)) @test isnothing(meteo_hint(m)) + @test meteo_inputs(m) == () + @test meteo_outputs(m) == () + @test meteo_inputs(ModelSpec(m)) == () + @test meteo_outputs(ModelSpec(m)) == () @test input_bindings(ModelSpec(m)) == NamedTuple() @test output_routing(ModelSpec(m)) == NamedTuple() @test model_scope(ModelSpec(m)) == :global + @test updates(ModelSpec(m)) == () + @test_throws "String scope selectors are not supported" ModelSpec(m; scope="plant") + @test_throws "String scope selectors are not supported" ScopeModel("plant")(m) + + scope_node = Node(MultiScaleTreeGraph.NodeMTG("/", :Leaf, 1, 1)) + @test_throws "must return `ScopeId` or `Symbol`" PlantSimEngine._scope_from_selector( + (node, scale, process) -> "plant", + scope_node, + :Leaf, + :process1, + ) mapping = Dict(:Leaf => (m,)) resolved_specs = resolved_model_specs(mapping) @@ -34,13 +50,19 @@ using Test TimeStepModel(24.0) |> InputBindings(; var1=(process=:process1, var=:var3)) |> OutputRouting(; var3=:stream_only) |> - ScopeModel(:plant) + ScopeModel(:plant) |> + Updates(:var3; after=:process1) |> + Updates(:var3; after=:process2) @test PlantSimEngine.model_(spec) === m @test PlantSimEngine.timestep(spec) == 24.0 @test input_bindings(spec).var1.process == :process1 @test input_bindings(spec).var1.policy isa HoldLast @test output_routing(spec).var3 == :stream_only @test model_scope(spec) == :plant + @test updates(spec)[1].variables == (:var3,) + @test updates(spec)[1].after == (:process1,) + @test updates(spec)[2].variables == (:var3,) + @test updates(spec)[2].after == (:process2,) mspec = ModelSpec(m) |> MultiScaleModel([:var1 => (:Leaf => :var1)]) @test length(PlantSimEngine.get_mapped_variables(mspec)) == 1 diff --git a/test/test-simulation.jl b/test/test-simulation.jl index c1a2fb8cc..f433c1c11 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -33,14 +33,15 @@ end; meteo = Atmosphere(T=20.0, Wind=1.0, Rh=0.65) run!(models, meteo) - @test_deprecated run!([models], meteo) + @test !isdefined(PlantSimEngine, :ModelList) + @test_throws MethodError run!([models], meteo) @test_throws ErrorException run!(ModelMapping("mod1" => models), meteo) mtg = Node(MultiScaleTreeGraph.NodeMTG("/", :Leaf, 1, 1)) mtg[:var1] = 15.0 mtg[:var2] = 0.3 mapping_dict = Dict(:Leaf => (Process1Model(1.0), Process2Model(), Process3Model())) - @test_deprecated run!(mtg, mapping_dict, meteo) + @test_throws ErrorException run!(mtg, mapping_dict, meteo) end @testset "Removed multirate keyword for single-scale" begin @@ -105,13 +106,13 @@ end; meteo = Atmosphere(T=20.0, Wind=1.0, Rh=0.65) @testset "simulation with an array of objects" begin - outputs_vector = run!([mapping, mapping2], meteo) + outputs_vector = [run!(m, meteo) for m in (mapping, mapping2)] @test [outputs_vector[1][i][1] for i in keys(outputs_vector[1])] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] @test [outputs_vector[2][i][1] for i in keys(outputs_vector[2])] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] end @testset "simulation with a dict of objects" begin - outputs_vector = run!(Dict("mod1" => mapping, "mod2" => mapping2), meteo) + outputs_vector = Dict("mod1" => run!(mapping, meteo), "mod2" => run!(mapping2, meteo)) @test [outputs_vector["mod1"][1][i] for i in keys(outputs_vector["mod1"])] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] @test [outputs_vector["mod2"][1][i] for i in keys(outputs_vector["mod2"])] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] end @@ -195,7 +196,7 @@ end; ) @testset "simulation with an array of objects" begin - outputs_vector = run!([mapping, mapping2], meteo) + outputs_vector = [run!(m, meteo) for m in (mapping, mapping2)] @test [outputs_vector[1][i] for i in keys(outputs_vector[1])] == [ [34.95, 40.0], [22.0, 23.2], [56.95, 63.2], [15.0, 16.0], [5.5, 5.8], [0.3, 0.3] ] @@ -205,7 +206,7 @@ end; end @testset "simulation with a dict of objects" begin - outputs_vector = run!(Dict("mod1" => mapping, "mod2" => mapping2), meteo) + outputs_vector = Dict("mod1" => run!(mapping, meteo), "mod2" => run!(mapping2, meteo)) @test [[outputs_vector["mod1"][1][i], outputs_vector["mod1"][2][i]] for i in keys(outputs_vector["mod1"])] == [ [34.95, 40.0], [22.0, 23.2], [56.95, 63.2], [15.0, 16.0], [5.5, 5.8], [0.3, 0.3] ] @@ -238,7 +239,7 @@ end; ) # var1 is taken from the MTG attributes but is a vector instead of a scalar, expecting an error: - VERSION >= v"1.8" && @test_throws AssertionError run!(mtg, mapping, meteo) + @test_throws ErrorException run!(mtg, mapping, meteo) leaf[:var1] = 15.0 diff --git a/test/test-unified-scene-object-api.jl b/test/test-unified-scene-object-api.jl new file mode 100644 index 000000000..4887a8254 --- /dev/null +++ b/test/test-unified-scene-object-api.jl @@ -0,0 +1,1372 @@ +using Dates +using PlantSimEngine +using PlantSimEngine.Examples +using Test + +PlantSimEngine.@process "scene_object_default_input_consumer" verbose = false + +struct SceneObjectDefaultInputConsumerModel <: AbstractScene_Object_Default_Input_ConsumerModel end + +PlantSimEngine.inputs_(::SceneObjectDefaultInputConsumerModel) = (leaf_carbon=[0.0],) +PlantSimEngine.outputs_(::SceneObjectDefaultInputConsumerModel) = (plant_carbon=0.0,) +PlantSimEngine.dep(::SceneObjectDefaultInputConsumerModel) = ( + leaf_carbon=Input(Many(scale=:Leaf, within=Self(), var=:leaf_carbon)), +) + +PlantSimEngine.@process "scene_object_default_call_consumer" verbose = false + +struct SceneObjectDefaultCallConsumerModel <: AbstractScene_Object_Default_Call_ConsumerModel end + +PlantSimEngine.inputs_(::SceneObjectDefaultCallConsumerModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectDefaultCallConsumerModel) = (energy_balance=0.0,) +PlantSimEngine.dep(::SceneObjectDefaultCallConsumerModel) = ( + stomata=Call(scale=:Leaf, process=:stomatal_conductance), +) + +PlantSimEngine.@process "scene_object_stomata" verbose = false + +struct SceneObjectStomataModel <: AbstractScene_Object_StomataModel end + +PlantSimEngine.inputs_(::SceneObjectStomataModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectStomataModel) = (gs=0.0,) + +PlantSimEngine.@process "scene_object_leaf_energy" verbose = false + +struct SceneObjectLeafEnergyModel <: AbstractScene_Object_Leaf_EnergyModel end + +PlantSimEngine.inputs_(::SceneObjectLeafEnergyModel) = (leaf_areas=[0.0],) +PlantSimEngine.outputs_(::SceneObjectLeafEnergyModel) = (leaf_temperature=25.0,) +PlantSimEngine.dep(::SceneObjectLeafEnergyModel) = ( + stomata=Call(process=:scene_object_stomata), +) + +PlantSimEngine.@process "scene_object_carrier_consumer" verbose = false + +struct SceneObjectCarrierConsumerModel <: AbstractScene_Object_Carrier_ConsumerModel end + +PlantSimEngine.inputs_(::SceneObjectCarrierConsumerModel) = (leaf_areas=[0.0], leaf_tokens=Any[]) +PlantSimEngine.outputs_(::SceneObjectCarrierConsumerModel) = (carrier_total=0.0,) + +function PlantSimEngine.run!(::SceneObjectCarrierConsumerModel, models, status, meteo, constants=nothing, extra=nothing) + status.carrier_total = sum(status.leaf_areas) + return nothing +end + +PlantSimEngine.@process "scene_object_environment_probe" verbose = false + +struct SceneObjectEnvironmentProbeModel <: AbstractScene_Object_Environment_ProbeModel end + +PlantSimEngine.inputs_(::SceneObjectEnvironmentProbeModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectEnvironmentProbeModel) = (temperature_seen=0.0,) +PlantSimEngine.meteo_inputs_(::SceneObjectEnvironmentProbeModel) = (T=0.0, CO2=0.0) + +function PlantSimEngine.run!(::SceneObjectEnvironmentProbeModel, models, status, meteo, constants=nothing, extra=nothing) + status.temperature_seen = meteo.T + return nothing +end + +PlantSimEngine.@process "scene_object_environment_update" verbose = false + +struct SceneObjectEnvironmentUpdateModel <: AbstractScene_Object_Environment_UpdateModel end + +PlantSimEngine.inputs_(::SceneObjectEnvironmentUpdateModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectEnvironmentUpdateModel) = (temperature_update=0.0,) +PlantSimEngine.meteo_inputs_(::SceneObjectEnvironmentUpdateModel) = (T=0.0,) +PlantSimEngine.meteo_outputs_(::SceneObjectEnvironmentUpdateModel) = (T=0.0,) + +function PlantSimEngine.run!(::SceneObjectEnvironmentUpdateModel, models, status, meteo, constants=nothing, extra=nothing) + status.temperature_update = meteo.T + 1.0 + status.T = status.temperature_update + return nothing +end + +PlantSimEngine.@process "scene_object_signal_source" verbose = false + +struct SceneObjectSignalSourceModel <: AbstractScene_Object_Signal_SourceModel end + +PlantSimEngine.inputs_(::SceneObjectSignalSourceModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectSignalSourceModel) = (signal=0.0,) + +function PlantSimEngine.run!(::SceneObjectSignalSourceModel, models, status, meteo, constants=nothing, extra=nothing) + status.signal += 1.0 + return nothing +end + +struct SceneObjectParameterizedSignalModel{T} <: AbstractScene_Object_Signal_SourceModel + increment::T +end + +PlantSimEngine.inputs_(::SceneObjectParameterizedSignalModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectParameterizedSignalModel) = (signal=0.0,) + +function PlantSimEngine.run!(model::SceneObjectParameterizedSignalModel, models, status, meteo, constants=nothing, extra=nothing) + status.signal += model.increment + return nothing +end + +PlantSimEngine.@process "scene_object_plant_signal_sum" verbose = false + +struct SceneObjectPlantSignalSumModel <: AbstractScene_Object_Plant_Signal_SumModel end + +PlantSimEngine.inputs_(::SceneObjectPlantSignalSumModel) = (signals=[0.0],) +PlantSimEngine.outputs_(::SceneObjectPlantSignalSumModel) = (signal_total=0.0,) + +function PlantSimEngine.run!(::SceneObjectPlantSignalSumModel, models, status, meteo, constants=nothing, extra=nothing) + status.signal_total = sum(status.signals) + return nothing +end + +PlantSimEngine.@process "scene_object_signal_caller" verbose = false + +struct SceneObjectSignalCallerModel <: AbstractScene_Object_Signal_CallerModel end + +PlantSimEngine.inputs_(::SceneObjectSignalCallerModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectSignalCallerModel) = (called_signal=0.0,) +PlantSimEngine.dep(::SceneObjectSignalCallerModel) = ( + signal=Call(process=:scene_object_signal_source), +) + +function PlantSimEngine.run!(::SceneObjectSignalCallerModel, models, status, meteo, constants=nothing, extra=nothing) + target = dependency_target(extra, :signal) + run_call!(target) + status.called_signal = target.status.signal + return nothing +end + +PlantSimEngine.@process "scene_object_signal_consumer" verbose = false + +struct SceneObjectSignalConsumerModel <: AbstractScene_Object_Signal_ConsumerModel end + +PlantSimEngine.inputs_(::SceneObjectSignalConsumerModel) = (signal=0.0,) +PlantSimEngine.outputs_(::SceneObjectSignalConsumerModel) = (observed_signal=0.0,) + +function PlantSimEngine.run!(::SceneObjectSignalConsumerModel, models, status, meteo, constants=nothing, extra=nothing) + status.observed_signal = status.signal + return nothing +end + +PlantSimEngine.@process "scene_object_cycle_a" verbose = false + +struct SceneObjectCycleAModel <: AbstractScene_Object_Cycle_AModel end + +PlantSimEngine.inputs_(::SceneObjectCycleAModel) = (cycle_b=0.0,) +PlantSimEngine.outputs_(::SceneObjectCycleAModel) = (cycle_a=0.0,) + +PlantSimEngine.@process "scene_object_cycle_b" verbose = false + +struct SceneObjectCycleBModel <: AbstractScene_Object_Cycle_BModel end + +PlantSimEngine.inputs_(::SceneObjectCycleBModel) = (cycle_a=0.0,) +PlantSimEngine.outputs_(::SceneObjectCycleBModel) = (cycle_b=0.0,) + +PlantSimEngine.@process "scene_object_temporal_sum" verbose = false + +struct SceneObjectTemporalSumModel <: AbstractScene_Object_Temporal_SumModel end + +PlantSimEngine.inputs_(::SceneObjectTemporalSumModel) = (signal_sum=0.0,) +PlantSimEngine.outputs_(::SceneObjectTemporalSumModel) = (temporal_total=0.0,) + +function PlantSimEngine.run!(::SceneObjectTemporalSumModel, models, status, meteo, constants=nothing, extra=nothing) + status.temporal_total = status.signal_sum + return nothing +end + +PlantSimEngine.@process "scene_object_biomass_source" verbose = false + +struct SceneObjectBiomassSourceModel <: AbstractScene_Object_Biomass_SourceModel end + +PlantSimEngine.inputs_(::SceneObjectBiomassSourceModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectBiomassSourceModel) = (biomass=0.0,) + +function PlantSimEngine.run!(::SceneObjectBiomassSourceModel, models, status, meteo, constants=nothing, extra=nothing) + status.biomass = 10.0 + return nothing +end + +PlantSimEngine.@process "scene_object_biomass_pruner" verbose = false + +struct SceneObjectBiomassPrunerModel <: AbstractScene_Object_Biomass_PrunerModel end + +PlantSimEngine.inputs_(::SceneObjectBiomassPrunerModel) = NamedTuple() +PlantSimEngine.outputs_(::SceneObjectBiomassPrunerModel) = (biomass=0.0,) + +function PlantSimEngine.run!(::SceneObjectBiomassPrunerModel, models, status, meteo, constants=nothing, extra=nothing) + status.biomass = 0.0 + return nothing +end + +mutable struct SceneObjectGridBackend <: PlantSimEngine.AbstractEnvironmentBackend + binds::Vector{Any} + index_updates::Vector{Any} +end + +SceneObjectGridBackend(binds::Vector{Any}=Any[]) = SceneObjectGridBackend(binds, Any[]) + +struct SceneObjectTaggedValue + value::Int +end + +PlantSimEngine.base_step_seconds(::SceneObjectGridBackend) = 3600.0 +PlantSimEngine.get_nsteps(::SceneObjectGridBackend) = 1 +PlantSimEngine.environment_variables(::SceneObjectGridBackend) = Set([:T, :CO2]) + +function PlantSimEngine.bind_environment( + backend::SceneObjectGridBackend, + object::Object, + support, + config, +) + object_geometry = geometry(object) + cell = isnothing(object_geometry) ? :global : object_geometry.cell + push!(backend.binds, (object=object.id.value, application=support.domain, cell=cell, config=config)) + return cell +end + +function PlantSimEngine.update_index!(backend::SceneObjectGridBackend, entities) + push!( + backend.index_updates, + [ + ( + id=entity.id, + scale=entity.scale, + kind=entity.kind, + geometry=entity.geometry, + position=entity.position, + bounds=entity.bounds, + ) + for entity in entities + ], + ) + return nothing +end + +mutable struct SceneObjectMutableEnvironmentBackend <: PlantSimEngine.AbstractEnvironmentBackend + values::Dict{Symbol,Float64} + cells_by_status::Dict{UInt,Symbol} + writes::Vector{Any} +end + +SceneObjectMutableEnvironmentBackend(values::Pair...) = + SceneObjectMutableEnvironmentBackend(Dict{Symbol,Float64}(values), Dict{UInt,Symbol}(), Any[]) + +PlantSimEngine.base_step_seconds(::SceneObjectMutableEnvironmentBackend) = 3600.0 +PlantSimEngine.get_nsteps(::SceneObjectMutableEnvironmentBackend) = 1 +PlantSimEngine.environment_variables(::SceneObjectMutableEnvironmentBackend) = Set([:T, :CO2]) + +function PlantSimEngine.bind_environment( + backend::SceneObjectMutableEnvironmentBackend, + object::Object, + support, + config, +) + cell = object.geometry.cell + backend.cells_by_status[objectid(object.status)] = cell + return cell +end + +function PlantSimEngine.sample( + backend::SceneObjectMutableEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + time, +) + variable == :CO2 && return 410.0 + variable == :T || error("Unexpected variable `$(variable)`.") + cell = backend.cells_by_status[objectid(support.status)] + return backend.values[cell] +end + +function PlantSimEngine.scatter!( + backend::SceneObjectMutableEnvironmentBackend, + variable::Symbol, + support::EnvironmentSupport, + value, + time, +) + variable == :T || error("Unexpected variable `$(variable)`.") + cell = backend.cells_by_status[objectid(support.status)] + backend.values[cell] = value + push!( + backend.writes, + ( + application=support.domain, + process=support.process, + cell=cell, + variable=variable, + value=value, + time=time, + ), + ) + return nothing +end + +@testset "Unified scene/object API" begin + scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, species=:oil_palm, parent=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1, geometry=(x=1.0, y=0.0)), + Object(:leaf_2; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1), + ) + + @test object_ids(scene; scale=:Leaf) == [ObjectId(:leaf_1), ObjectId(:leaf_2)] + @test object_ids(scene; kind=:plant, species=:oil_palm) == [ObjectId(:leaf_1), ObjectId(:leaf_2), ObjectId(:plant_1)] + @test only(scene_objects(scene; scale=:Scene)).id == ObjectId(:scene) + + leaf_2 = move_object!(scene, :leaf_2, (x=2.0, y=0.0)) + @test leaf_2.geometry == (x=2.0, y=0.0) + @test geometry(leaf_2) == (x=2.0, y=0.0) + @test position(leaf_2) == (x=2.0, y=0.0) + @test isnothing(bounds(leaf_2)) + bounded_leaf = Object(:bounded_leaf; scale=:Leaf, geometry=(position=(x=1.0, y=2.0, z=3.0), bounds=(radius=0.5,))) + @test position(bounded_leaf) == (x=1.0, y=2.0, z=3.0) + @test bounds(bounded_leaf) == (radius=0.5,) + + new_axis = register_object!(scene, Object(:axis_1; scale=:Axis, kind=:plant, species=:oil_palm); parent=:plant_1) + @test new_axis.parent == ObjectId(:plant_1) + @test ObjectId(:axis_1) in only(scene_objects(scene; scale=:Plant)).children + + reparent_object!(scene, :leaf_2, :axis_1) + @test only(scene_objects(scene; name=nothing, scale=:Axis)).children == [ObjectId(:leaf_2)] + @test ObjectId(:leaf_2) ∉ only(scene_objects(scene; scale=:Plant)).children + + removed_axis = remove_object!(scene, :axis_1) + @test removed_axis.id == ObjectId(:axis_1) + @test object_ids(scene; scale=:Axis) == ObjectId[] + @test object_ids(scene; name=:leaf_2) == ObjectId[] + + object_rows = explain_objects(scene) + @test length(object_rows) == 3 + @test any(row -> row.id == :leaf_1 && row.has_geometry, object_rows) + @test any(row -> row.id == :plant_1 && row.children == [ObjectId(:leaf_1).value], object_rows) + + selector_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, species=:oil_palm, name=:palm_1, parent=:scene), + Object(:axis_1; scale=:Axis, kind=:plant, species=:oil_palm, parent=:plant_1), + Object(:leaf_1; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1), + Object(:leaf_2; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:axis_1), + Object(:plant_2; scale=:Plant, kind=:plant, species=:oil_palm, name=:palm_2, parent=:scene), + Object(:leaf_3; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_2), + Object(:soil; scale=:Soil, kind=:soil, parent=:scene), + ) + + scope_rows = explain_scopes(selector_scene) + scene_scope = only(row for row in scope_rows if row.scope_type == :scene) + @test scene_scope.selector isa SceneScope + @test scene_scope.object_ids == [:axis_1, :leaf_1, :leaf_2, :leaf_3, :plant_1, :plant_2, :scene, :soil] + plant_1_scope = only(row for row in scope_rows if row.scope_type == :object_subtree && row.root_id == :plant_1) + @test plant_1_scope.selector isa Self + @test plant_1_scope.object_ids == [:axis_1, :leaf_1, :leaf_2, :plant_1] + palm_2_scope = only(row for row in scope_rows if row.scope_type == :named_scope && row.name == :palm_2) + @test palm_2_scope.selector isa Scope + @test palm_2_scope.root_id == :plant_2 + @test palm_2_scope.object_ids == [:leaf_3, :plant_2] + leaf_label_scope = only(row for row in scope_rows if row.scope_type == :scale && row.scale == :Leaf) + @test leaf_label_scope.selector == (:scale => :Leaf) + @test leaf_label_scope.object_ids == [:leaf_1, :leaf_2, :leaf_3] + oil_palm_scope = only(row for row in scope_rows if row.scope_type == :species && row.species == :oil_palm) + @test oil_palm_scope.object_ids == [:axis_1, :leaf_1, :leaf_2, :leaf_3, :plant_1, :plant_2] + + @test resolve_object_ids(selector_scene, Many(scale=:Leaf)) == + [ObjectId(:leaf_1), ObjectId(:leaf_2), ObjectId(:leaf_3)] + @test only(resolve_objects(selector_scene, One(scale=:Scene))).id == ObjectId(:scene) + @test resolve_object_ids(selector_scene, Many(Kind(:plant), Scale(:Leaf))) == + [ObjectId(:leaf_1), ObjectId(:leaf_2), ObjectId(:leaf_3)] + @test resolve_object_ids(selector_scene, Many(scale=:Leaf, within=Self()); context=:plant_1) == + [ObjectId(:leaf_1), ObjectId(:leaf_2)] + @test resolve_object_ids(selector_scene, Many(scale=:Leaf, within=Self()); context=:leaf_2) == + [ObjectId(:leaf_2)] + @test resolve_object_ids(selector_scene, Many(scale=:Leaf, within=SelfPlant()); context=:leaf_2) == + [ObjectId(:leaf_1), ObjectId(:leaf_2)] + @test resolve_object_ids(selector_scene, Many(scale=:Leaf, within=Ancestor(scale=:Axis)); context=:leaf_2) == + [ObjectId(:leaf_2)] + @test resolve_object_ids(selector_scene, Many(scale=:Leaf, within=Scope(:palm_2))) == + [ObjectId(:leaf_3)] + @test resolve_object_ids(selector_scene, OptionalOne(scale=:Flower)) == ObjectId[] + @test_throws ErrorException resolve_object_ids(selector_scene, One(scale=:Flower)) + @test_throws ErrorException resolve_object_ids(selector_scene, One(scale=:Leaf)) + @test_throws ErrorException resolve_object_ids(selector_scene, Many(scale=:Leaf, within=Self())) + @test resolve_object_ids(selector_scene, Many(scale=:Leaf); context=:plant_1) == + [ObjectId(:leaf_1), ObjectId(:leaf_2), ObjectId(:leaf_3)] + + shared_signal_model = SceneObjectParameterizedSignalModel(1.0) + shared_template_parameters = Dict(:signal_increment => 1.0) + plant_template = ObjectTemplate( + ( + ModelSpec(shared_signal_model; name=:signal_source) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectPlantSignalSumModel(); name=:plant_total) |> + AppliesTo(One(scale=:Plant)) |> + Inputs( + :signals => Many( + scale=:Leaf, + within=Self(), + process=:scene_object_signal_source, + var=:signal, + ), + ), + ); + kind=:plant, + species=:oil_palm, + parameters=shared_template_parameters, + ) + palm_1_leaf_override = SceneObjectParameterizedSignalModel(3.0) + palm_1 = ObjectInstance( + :palm_1, + plant_template; + root=Object(:templated_plant_1; scale=:Plant, parent=:scene, status=Status(signals=[0.0], signal_total=0.0)), + objects=( + Object(:templated_leaf_1; scale=:Leaf, parent=:templated_plant_1, status=Status(signal=0.0)), + Object(:templated_leaf_1_exception; scale=:Leaf, parent=:templated_plant_1, status=Status(signal=0.0)), + ), + object_overrides=( + Override( + object=:templated_leaf_1_exception, + application=:signal_source, + model=palm_1_leaf_override, + ), + ), + ) + palm_2_override = SceneObjectParameterizedSignalModel(2.0) + palm_2 = ObjectInstance( + :palm_2, + plant_template; + root=Object(:templated_plant_2; scale=:Plant, parent=:scene, status=Status(signals=[0.0], signal_total=0.0)), + objects=(Object(:templated_leaf_2; scale=:Leaf, parent=:templated_plant_2, status=Status(signal=0.0)),), + overrides=(scene_object_signal_source=palm_2_override,), + ) + palm_3 = ObjectInstance( + :palm_3, + plant_template; + root=Object(:templated_plant_3; scale=:Plant, parent=:scene, status=Status(signals=[0.0], signal_total=0.0)), + objects=(Object(:templated_leaf_3; scale=:Leaf, parent=:templated_plant_3, status=Status(signal=0.0)),), + ) + templated_plant_4 = Object( + :templated_plant_4; + scale=:Plant, + parent=:scene, + status=Status(signals=[0.0], signal_total=0.0), + ) + templated_leaf_4 = Object( + :templated_leaf_4; + scale=:Leaf, + parent=:templated_plant_4, + status=Status(signal=0.0), + ) + palm_4 = ObjectInstance(:palm_4, plant_template; root=:templated_plant_4) + template_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + templated_plant_4, + templated_leaf_4, + palm_1, + palm_2, + palm_3; + instances=(palm_4,), + ) + @test length(template_scene.applications) == 8 + @test plant_template.parameters === shared_template_parameters + @test only(scene_objects(template_scene; name=:palm_1)).id == ObjectId(:templated_plant_1) + @test object_ids(template_scene; species=:oil_palm) == [ + ObjectId(:templated_leaf_1), + ObjectId(:templated_leaf_1_exception), + ObjectId(:templated_leaf_2), + ObjectId(:templated_leaf_3), + ObjectId(:templated_leaf_4), + ObjectId(:templated_plant_1), + ObjectId(:templated_plant_2), + ObjectId(:templated_plant_3), + ObjectId(:templated_plant_4), + ] + template_compiled = compile_scene(template_scene) + template_application_rows = explain_scene_applications(template_compiled) + @test only(row for row in template_application_rows if row.application_id == :palm_1__signal_source).target_ids == + [:templated_leaf_1, :templated_leaf_1_exception] + @test only(row for row in template_application_rows if row.application_id == :palm_2__signal_source).target_ids == + [:templated_leaf_2] + @test only(row for row in template_application_rows if row.application_id == :palm_3__plant_total).target_ids == + [:templated_plant_3] + palm_1_signal_row = only( + row for row in template_application_rows + if row.application_id == :palm_1__signal_source + ) + @test palm_1_signal_row.model_type == typeof(shared_signal_model) + @test palm_1_signal_row.model_storage == :per_object_override + @test palm_1_signal_row.model_dispatch == :concrete_per_object + @test palm_1_signal_row.object_overrides == [ + ( + object_id=:templated_leaf_1_exception, + model_type=typeof(palm_1_leaf_override), + ), + ] + @test (@inferred PlantSimEngine._application_model( + template_compiled.applications_by_id[:palm_1__signal_source], + ObjectId(:templated_leaf_1), + )) === shared_signal_model + @test (@inferred PlantSimEngine._application_model( + template_compiled.applications_by_id[:palm_1__signal_source], + ObjectId(:templated_leaf_1_exception), + )) === palm_1_leaf_override + @test PlantSimEngine.model_(template_compiled.applications_by_id[:palm_3__signal_source].spec) === shared_signal_model + @test PlantSimEngine.model_(template_compiled.applications_by_id[:palm_4__signal_source].spec) === shared_signal_model + @test PlantSimEngine.model_(template_compiled.applications_by_id[:palm_2__signal_source].spec) === palm_2_override + template_instance_rows = explain_instances(template_scene) + palm_1_instance_row = only(row for row in template_instance_rows if row.name == :palm_1) + @test palm_1_instance_row.root_id == :templated_plant_1 + @test palm_1_instance_row.object_ids == + [:templated_leaf_1, :templated_leaf_1_exception, :templated_plant_1] + @test palm_1_instance_row.application_ids == + [:palm_1__plant_total, :palm_1__signal_source] + @test palm_1_instance_row.object_overrides == [ + ( + object_id=:templated_leaf_1_exception, + process=nothing, + application=:signal_source, + model_type=typeof(palm_1_leaf_override), + ), + ] + @test palm_1_instance_row.parameters_shared_by_reference + @test only(row for row in explain_objects(template_scene) if row.id == :templated_leaf_1).instance == + :palm_1 + run!(template_scene; steps=1) + @test only(scene_objects(template_scene; name=:palm_1)).status.signal_total == 4.0 + @test only(scene_objects(template_scene; name=:palm_2)).status.signal_total == 2.0 + @test only(scene_objects(template_scene; name=:palm_3)).status.signal_total == 1.0 + @test only(scene_objects(template_scene; name=:palm_4)).status.signal_total == 1.0 + registered_template_leaf = register_object!( + template_scene, + Object(:templated_leaf_new; scale=:Leaf, status=Status(signal=0.0)); + parent=:templated_plant_2, + ) + @test registered_template_leaf.kind == :plant + @test registered_template_leaf.species == :oil_palm + @test :templated_leaf_new in only( + row.object_ids for row in explain_instances(template_scene) + if row.name == :palm_2 + ) + refreshed_template = refresh_bindings!(template_scene) + @test ObjectId(:templated_leaf_new) in + refreshed_template.applications_by_id[:palm_2__signal_source].target_ids + remove_object!(template_scene, :templated_leaf_new) + @test_throws ErrorException Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :invalid_palm, + plant_template; + root=Object(:invalid_plant; scale=:Plant, parent=:scene), + overrides=(missing_process=shared_signal_model,), + ), + ) + @test_throws ErrorException Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :invalid_palm, + plant_template; + root=Object(:invalid_plant; scale=:Plant, parent=:scene), + overrides=(signal_source=Process1Model(1.0),), + ), + ) + @test_throws ErrorException Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :invalid_palm, + plant_template; + root=Object(:invalid_plant; scale=:Plant, parent=:scene), + objects=(Object(:invalid_leaf; scale=:Leaf, parent=:invalid_plant),), + object_overrides=( + Override( + object=:outside_instance, + application=:signal_source, + model=shared_signal_model, + ), + ), + ), + ) + @test_throws ErrorException Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :invalid_palm, + plant_template; + root=Object(:invalid_plant; scale=:Plant, parent=:scene), + objects=(Object(:invalid_leaf; scale=:Leaf, parent=:invalid_plant),), + object_overrides=( + Override( + object=:invalid_leaf, + application=:signal_source, + model=shared_signal_model, + ), + Override( + object=:invalid_leaf, + process=:scene_object_signal_source, + model=shared_signal_model, + ), + ), + ), + ) + unmatched_override_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :invalid_palm, + plant_template; + root=Object(:invalid_plant; scale=:Plant, parent=:scene), + objects=(Object(:invalid_leaf; scale=:Leaf, parent=:invalid_plant, status=Status(signal=0.0)),), + object_overrides=( + Override( + object=:invalid_plant, + application=:signal_source, + model=shared_signal_model, + ), + ), + ), + ) + @test_throws ErrorException compile_scene(unmatched_override_scene) + + call_template = ObjectTemplate( + ( + ModelSpec(shared_signal_model; name=:signal_source) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectSignalCallerModel(); name=:signal_caller) |> + AppliesTo(Many(scale=:Leaf)), + ); + kind=:plant, + species=:oil_palm, + ) + call_override_model = SceneObjectParameterizedSignalModel(4.0) + call_override_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + ObjectInstance( + :call_palm, + call_template; + root=Object(:call_plant; scale=:Plant, parent=:scene), + objects=( + Object(:call_leaf_1; scale=:Leaf, parent=:call_plant, status=Status(signal=0.0, called_signal=0.0)), + Object(:call_leaf_2; scale=:Leaf, parent=:call_plant, status=Status(signal=0.0, called_signal=0.0)), + ), + object_overrides=( + Override( + object=:call_leaf_2, + application=:signal_source, + model=call_override_model, + ), + ), + ), + ) + run!(call_override_scene) + call_leaf_1 = only(object for object in scene_objects(call_override_scene; scale=:Leaf) if object.id == ObjectId(:call_leaf_1)) + call_leaf_2 = only(object for object in scene_objects(call_override_scene; scale=:Leaf) if object.id == ObjectId(:call_leaf_2)) + @test call_leaf_1.status.called_signal == 1.0 + @test call_leaf_2.status.called_signal == 4.0 + + leaf_selector = Many( + kind="plant", + scale=:Leaf, + within=Self(), + process="leaf_state", + var="leaf_area", + policy=Integrate(), + window=Day(1), + ) + + @test leaf_selector.criteria.kind == :plant + @test leaf_selector.criteria.scale == :Leaf + @test leaf_selector.criteria.within isa Self + @test leaf_selector.criteria.process == :leaf_state + @test leaf_selector.criteria.var == :leaf_area + @test leaf_selector.criteria.policy isa Integrate + @test leaf_selector.criteria.window == Day(1) + + address = object_address(leaf_selector) + @test address.scope isa Self + @test address.kind == :plant + @test address.scale == :Leaf + @test address.process == :leaf_state + @test address.var == :leaf_area + @test address.multiplicity == :many + + @test One(Kind(:plant), Scale(:Leaf)).criteria.selectors == (Kind(:plant), Scale(:Leaf)) + @test object_address(OptionalOne(scale=:Scene)).multiplicity == :optional_one + + default_input = Input(Many(scale=:Leaf, within=Self(), var=:leaf_carbon)) + @test default_input.selector isa Many + @test default_input.selector.criteria.within isa Self + + default_call = Call(process=:stomatal_conductance) + @test default_call.selector isa One + @test object_address(default_call.selector).process == :stomatal_conductance + + m = Process1Model(1.0) + spec = ModelSpec(m; name=:leaf_energy) |> + AppliesTo(Many(kind=:plant, scale=:Leaf)) |> + Inputs( + :leaf_areas => Many(kind=:plant, scale=:Leaf, within=SceneScope(), var=:leaf_area), + :leaf_carbon => Many(scale=:Leaf, within=Self(), var=:leaf_carbon, policy=Integrate(), window=Day(1)), + ) |> + Calls(:stomata => One(scale=:Leaf, process=:stomatal_conductance)) |> + TimeStep(Hour(1)) |> + Environment(provider=:global) + + @test PlantSimEngine.model_(spec) === m + @test application_name(spec) == :leaf_energy + @test applies_to(spec) isa Many + @test applies_to(spec).criteria.kind == :plant + @test value_inputs(spec).leaf_areas isa Many + @test value_inputs(spec).leaf_carbon.criteria.policy isa Integrate + @test value_inputs(spec).leaf_carbon.criteria.window == Day(1) + @test model_calls(spec).stomata isa One + @test object_address(model_calls(spec).stomata).process == :stomatal_conductance + call_dep = dep(spec).stomata + @test call_dep isa HardDomains + @test call_dep.scale == :Leaf + @test call_dep.process == :stomatal_conductance + @test PlantSimEngine.timestep(spec) == Hour(1) + @test environment_config(spec) isa PlantSimEngine.EnvironmentConfig + @test environment_config(spec).config.provider == :global + + # The old multirate metadata stays available while the compiler is migrated. + legacy_and_unified = spec |> + InputBindings(; var1=(process=:process1, var=:var3)) |> + OutputRouting(; var3=:stream_only) |> + ScopeModel(:plant) |> + Updates(:var3; after=:process1) + @test input_bindings(legacy_and_unified).var1.process == :process1 + @test output_routing(legacy_and_unified).var3 == :stream_only + @test model_scope(legacy_and_unified) == :plant + @test updates(legacy_and_unified)[1].after == (:process1,) + @test value_inputs(legacy_and_unified) == value_inputs(spec) + @test model_calls(legacy_and_unified) == model_calls(spec) + + rows = explain_model_specs(Dict(:Leaf => (spec,)); io=IOBuffer()) + @test length(rows) == 1 + @test rows[1].application_name == :leaf_energy + @test rows[1].applies_to === applies_to(spec) + @test rows[1].value_inputs == value_inputs(spec) + @test rows[1].model_calls == model_calls(spec) + @test rows[1].environment === environment_config(spec) + + leaf_assim = ModelSpec(ToyAssimModel()) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs(:soil_water_content => One(scale=:Soil, var=:soil_water_content)) + @test length(PlantSimEngine.get_mapped_variables(leaf_assim)) == 1 + + mapping = Dict( + :Leaf => (leaf_assim,), + :Soil => (ToySoilWaterModel(),), + ) + resolved = resolved_model_specs(mapping) + binding = input_bindings(resolved[:Leaf][:carbon_assimilation]).soil_water_content + @test binding.scale == :Soil + @test binding.var == :soil_water_content + @test binding.process == :soil_water + + rich_selector_spec = ModelSpec(ToyAssimModel()) |> + Inputs(:soil_water_content => One(kind=:soil, scale=:Soil, var=:soil_water_content)) + @test isempty(PlantSimEngine.get_mapped_variables(rich_selector_spec)) + @test value_inputs(rich_selector_spec).soil_water_content.criteria.kind == :soil + + default_input_spec = ModelSpec(SceneObjectDefaultInputConsumerModel()) + @test value_inputs(default_input_spec).leaf_carbon isa Many + @test value_inputs(default_input_spec).leaf_carbon.criteria.within isa Self + @test !haskey(dep(default_input_spec), :leaf_carbon) + @test length(PlantSimEngine.get_mapped_variables(default_input_spec)) == 1 + + override_input_spec = ModelSpec(SceneObjectDefaultInputConsumerModel()) |> + Inputs(:leaf_carbon => Many(scale=:Leaf, var=:carbon_override)) + @test value_inputs(override_input_spec).leaf_carbon.criteria.var == :carbon_override + mapped = only(PlantSimEngine.get_mapped_variables(override_input_spec)) + @test first(mapped) == :leaf_carbon + @test last(mapped) == [:Leaf => :carbon_override] + + default_call_spec = ModelSpec(SceneObjectDefaultCallConsumerModel()) + @test model_calls(default_call_spec).stomata isa One + @test model_calls(default_call_spec).stomata.criteria.scale == :Leaf + @test model_calls(default_call_spec).stomata.criteria.process == :stomatal_conductance + default_call_dep = dep(default_call_spec).stomata + @test default_call_dep isa HardDomains + @test default_call_dep.scale == :Leaf + @test default_call_dep.process == :stomatal_conductance + + override_call_spec = ModelSpec(SceneObjectDefaultCallConsumerModel()) |> + Calls(:stomata => One(scale=:Internode, process=:water_status)) + @test model_calls(override_call_spec).stomata.criteria.scale == :Internode + @test model_calls(override_call_spec).stomata.criteria.process == :water_status + override_call_dep = dep(override_call_spec).stomata + @test override_call_dep isa HardDomains + @test override_call_dep.scale == :Internode + @test override_call_dep.process == :water_status + + compiled_specs = ( + ModelSpec(SceneObjectStomataModel(); name=:stomata) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectLeafEnergyModel(); name=:leaf_energy) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs(:leaf_areas => Many(scale=:Leaf, within=SelfPlant(), var=:leaf_area, policy=Integrate(), window=Day(1))), + ) + compiled = compile_scene(selector_scene, compiled_specs) + application_rows = explain_scene_applications(compiled) + @test length(application_rows) == 2 + @test only(row for row in application_rows if row.application_id == :stomata).target_ids == + [:leaf_1, :leaf_2, :leaf_3] + @test only(row for row in application_rows if row.application_id == :leaf_energy).target_ids == + [:leaf_1, :leaf_2, :leaf_3] + + binding_rows = explain_bindings(compiled) + @test length(binding_rows) == 3 + leaf_2_binding = only(row for row in binding_rows if row.consumer_id == :leaf_2) + @test leaf_2_binding.application_id == :leaf_energy + @test leaf_2_binding.input == :leaf_areas + @test leaf_2_binding.source_ids == [:leaf_1, :leaf_2] + @test leaf_2_binding.source_var == :leaf_area + @test leaf_2_binding.carrier_hint == :temporal_stream + @test leaf_2_binding.carrier_kind == :temporal_stream + @test leaf_2_binding.copy_semantics == :materialized_temporal_value + + call_rows = explain_calls(compiled) + @test length(call_rows) == 3 + leaf_2_call = only(row for row in call_rows if row.consumer_id == :leaf_2) + @test leaf_2_call.application_id == :leaf_energy + @test leaf_2_call.call == :stomata + @test leaf_2_call.callee_object_ids == [:leaf_2] + @test leaf_2_call.callee_application_ids == [:stomata] + @test leaf_2_call.process == :scene_object_stomata + + ambiguous_call_specs = ( + ModelSpec(SceneObjectStomataModel(); name=:sunlit_stomata) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectStomataModel(); name=:shaded_stomata) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectLeafEnergyModel(); name=:leaf_energy) |> + AppliesTo(Many(scale=:Leaf)), + ) + @test_throws ErrorException compile_scene(selector_scene, ambiguous_call_specs) + + disambiguated_call_specs = ( + ModelSpec(SceneObjectStomataModel(); name=:sunlit_stomata) |> + AppliesTo(Many(scale=:Leaf)), + ModelSpec(SceneObjectStomataModel(); name=:shaded_stomata) |> + AppliesTo(Many(scale=:Leaf)) |> + Updates(:gs; after=:sunlit_stomata), + ModelSpec(SceneObjectLeafEnergyModel(); name=:leaf_energy) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs(:leaf_areas => Many(scale=:Leaf, within=SelfPlant(), var=:leaf_area)) |> + Calls(:stomata => One(process=:scene_object_stomata, application=:sunlit_stomata)), + ) + disambiguated = compile_scene(selector_scene, disambiguated_call_specs) + disambiguated_call = only(row for row in explain_calls(disambiguated) if row.consumer_id == :leaf_2) + @test disambiguated_call.callee_application_ids == [:sunlit_stomata] + @test disambiguated_call.application == :sunlit_stomata + leaf_2_call_bindings = disambiguated.call_bindings_by_target[(:leaf_energy, ObjectId(:leaf_2))] + @test length(leaf_2_call_bindings) == 1 + @test only(leaf_2_call_bindings).call == :stomata + + default_scope_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene, status=Status(signal_sum=0.0, temporal_total=0.0)), + Object(:plant_1; scale=:Plant, kind=:plant, parent=:scene, status=Status(signal_sum=0.0, temporal_total=0.0)), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:plant_1, status=Status(leaf_area=1.0)), + Object(:leaf_2; scale=:Leaf, kind=:plant, parent=:plant_1, status=Status(leaf_area=2.0)), + Object(:plant_2; scale=:Plant, kind=:plant, parent=:scene, status=Status(signal_sum=0.0, temporal_total=0.0)), + Object(:leaf_3; scale=:Leaf, kind=:plant, parent=:plant_2, status=Status(leaf_area=3.0)), + ) + plant_default_scope = compile_scene( + default_scope_scene, + ( + ModelSpec(SceneObjectTemporalSumModel(); name=:plant_leaf_sum) |> + AppliesTo(Many(scale=:Plant)) |> + Inputs(:signal_sum => Many(scale=:Leaf, var=:leaf_area)), + ), + ) + @test only(row for row in explain_bindings(plant_default_scope) if row.consumer_id == :plant_1).source_ids == + [:leaf_1, :leaf_2] + @test only(row for row in explain_bindings(plant_default_scope) if row.consumer_id == :plant_2).source_ids == + [:leaf_3] + scene_default_scope = compile_scene( + default_scope_scene, + ( + ModelSpec(SceneObjectTemporalSumModel(); name=:scene_leaf_sum) |> + AppliesTo(One(scale=:Scene)) |> + Inputs(:signal_sum => Many(scale=:Leaf, var=:leaf_area)), + ), + ) + @test only(explain_bindings(scene_default_scope)).source_ids == [:leaf_1, :leaf_2, :leaf_3] + + inferred_input_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0, observed_signal=0.0)), + ) + inferred_input_specs = ( + ModelSpec(SceneObjectSignalSourceModel(); name=:signal_source) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)), + ) + inferred_compiled = compile_scene(inferred_input_scene, inferred_input_specs) + inferred_binding = only(explain_bindings(inferred_compiled)) + @test inferred_binding.application_id == :signal_consumer + @test inferred_binding.input == :signal + @test inferred_binding.origin == :inferred_same_object + @test inferred_binding.source_ids == [:leaf_1] + @test inferred_binding.source_application_ids == [:signal_source] + @test inferred_binding.process == :scene_object_signal_source + @test inferred_binding.application == :signal_source + @test inferred_binding.has_reference_carrier + @test inferred_binding.carrier_kind == :ref + @test inferred_binding.copy_semantics == :live_references + inferred_input_scene_with_apps = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0, observed_signal=0.0)); + applications=inferred_input_specs, + ) + run!(inferred_input_scene_with_apps) + @test only(scene_objects(inferred_input_scene_with_apps; scale=:Leaf)).status.observed_signal == 1.0 + + reversed_dependency_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0, observed_signal=0.0)); + applications=reverse(inferred_input_specs), + ) + reversed_compiled = refresh_bindings!(reversed_dependency_scene) + @test length(reversed_compiled.applications_by_id) == length(reversed_compiled.applications) + @test reversed_compiled.applications_by_id[:signal_source].process == :scene_object_signal_source + @test reversed_compiled.applications_by_id[:signal_consumer].process == :scene_object_signal_consumer + @test reversed_compiled.application_order == [:signal_source, :signal_consumer] + @test [row.application_id for row in explain_schedule(reversed_compiled)] == + [:signal_source, :signal_consumer] + @test [row.execution_index for row in explain_schedule(reversed_compiled)] == [1, 2] + run!(reversed_dependency_scene) + @test only(scene_objects(reversed_dependency_scene; scale=:Leaf)).status.observed_signal == 1.0 + + @test_throws ErrorException compile_scene( + Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(cycle_a=0.0, cycle_b=0.0)), + ), + ( + ModelSpec(SceneObjectCycleAModel(); name=:cycle_a) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectCycleBModel(); name=:cycle_b) |> + AppliesTo(One(scale=:Leaf)), + ), + ) + + @test_throws ErrorException compile_scene( + Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(observed_signal=0.0)), + ), + ( + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)), + ), + ) + @test_throws ErrorException compile_scene( + inferred_input_scene, + ( + ModelSpec(SceneObjectSignalSourceModel(); name=:sunlit_signal) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalSourceModel(); name=:shaded_signal) |> + AppliesTo(One(scale=:Leaf)) |> + Updates(:signal; after=:sunlit_signal), + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)), + ), + ) + + filtered_input_specs = ( + ModelSpec(SceneObjectSignalSourceModel(); name=:signal_source) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)) |> + Inputs(:signal => One(scale=:Leaf, var=:signal, process=:scene_object_signal_source, application=:signal_source)), + ) + filtered_binding = only(explain_bindings(compile_scene(inferred_input_scene, filtered_input_specs))) + @test filtered_binding.origin == :declared + @test filtered_binding.source_application_ids == [:signal_source] + @test filtered_binding.process == :scene_object_signal_source + @test filtered_binding.application == :signal_source + @test_throws ErrorException compile_scene( + inferred_input_scene, + ( + filtered_input_specs[1], + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)) |> + Inputs(:signal => One(scale=:Leaf, var=:signal, application=:missing_source)), + ), + ) + @test_throws ErrorException compile_scene( + inferred_input_scene, + ( + filtered_input_specs[1], + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)) |> + Inputs(:siggnal => One(scale=:Leaf, var=:signal, application=:signal_source)), + ), + ) + @test_throws ErrorException compile_scene( + inferred_input_scene, + ( + filtered_input_specs[1], + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)) |> + Inputs(:signal => One(scale=:Leaf, var=:missing_signal)), + ), + ) + + carrier_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, species=:oil_palm, parent=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1, status=Status(leaf_area=1.0, leaf_token=SceneObjectTaggedValue(1), aPPFD=100.0)), + Object(:leaf_2; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1, status=Status(leaf_area=2.0, leaf_token=2, aPPFD=100.0)), + Object(:soil; scale=:Soil, kind=:soil, parent=:scene, status=Status(soil_water_content=0.31)), + ) + carrier_specs = ( + ModelSpec(SceneObjectCarrierConsumerModel(); name=:carrier_consumer) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs( + :leaf_areas => Many(scale=:Leaf, within=SelfPlant(), var=:leaf_area), + :leaf_tokens => Many(scale=:Leaf, within=SelfPlant(), var=:leaf_token), + ), + ModelSpec(ToyAssimModel(); name=:assim) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs(:soil_water_content => One(scale=:Soil, within=SceneScope(), var=:soil_water_content)), + ) + carrier_compiled = compile_scene(carrier_scene, carrier_specs) + carrier_rows = explain_bindings(carrier_compiled) + leaf_1_carrier_bindings = carrier_compiled.input_bindings_by_target[(:carrier_consumer, ObjectId(:leaf_1))] + @test length(leaf_1_carrier_bindings) == 2 + @test Set(binding.input for binding in leaf_1_carrier_bindings) == Set((:leaf_areas, :leaf_tokens)) + @test length(carrier_compiled.input_bindings_by_target[(:assim, ObjectId(:leaf_1))]) == 1 + leaf_area_binding = only( + binding for binding in carrier_compiled.input_bindings + if binding.application_id == :carrier_consumer && binding.consumer_id == ObjectId(:leaf_1) && binding.input == :leaf_areas + ) + @test has_reference_carrier(leaf_area_binding) + @test input_carrier(leaf_area_binding) isa PlantSimEngine.RefVector + @test input_value(leaf_area_binding)[1] == 1.0 + input_value(leaf_area_binding)[1] = 4.0 + leaf_1_object = only(object for object in scene_objects(carrier_scene; scale=:Leaf) if object.id == ObjectId(:leaf_1)) + @test leaf_1_object.status.leaf_area == 4.0 + leaf_area_row = only(row for row in carrier_rows if row.application_id == :carrier_consumer && row.consumer_id == :leaf_1 && row.input == :leaf_areas) + @test leaf_area_row.has_reference_carrier + @test leaf_area_row.carrier_kind == :ref_vector + @test leaf_area_row.copy_semantics == :live_references + + token_binding = only( + binding for binding in carrier_compiled.input_bindings + if binding.application_id == :carrier_consumer && binding.consumer_id == ObjectId(:leaf_1) && binding.input == :leaf_tokens + ) + @test input_value(token_binding)[2] == 2 + input_value(token_binding)[2] = 20 + leaf_2_object = only(object for object in scene_objects(carrier_scene; scale=:Leaf) if object.id == ObjectId(:leaf_2)) + @test leaf_2_object.status.leaf_token == 20 + token_row = only(row for row in carrier_rows if row.application_id == :carrier_consumer && row.consumer_id == :leaf_1 && row.input == :leaf_tokens) + @test token_row.carrier_kind == :object_ref_vector + @test token_row.copy_semantics == :live_references + + scalar_binding = only( + binding for binding in carrier_compiled.input_bindings + if binding.application_id == :assim && binding.consumer_id == ObjectId(:leaf_1) + ) + @test has_reference_carrier(scalar_binding) + @test input_carrier(scalar_binding) isa Base.RefValue + @test input_value(scalar_binding) == 0.31 + input_carrier(scalar_binding)[] = 0.42 + @test only(scene_objects(carrier_scene; scale=:Soil)).status.soil_water_content == 0.42 + scalar_row = only(row for row in carrier_rows if row.application_id == :assim && row.consumer_id == :leaf_1) + @test scalar_row.carrier_kind == :ref + @test scalar_row.copy_semantics == :live_references + + cache_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, species=:oil_palm, name=:palm_1, parent=:scene), + Object(:axis_1; scale=:Axis, kind=:plant, species=:oil_palm, parent=:plant_1), + Object(:leaf_1; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1), + Object(:leaf_2; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:axis_1), + Object(:plant_2; scale=:Plant, kind=:plant, species=:oil_palm, name=:palm_2, parent=:scene), + Object(:leaf_3; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_2), + Object(:soil; scale=:Soil, kind=:soil, parent=:scene); + applications=compiled_specs, + ) + @test bindings_dirty(cache_scene) + cached_a = refresh_bindings!(cache_scene) + @test cached_a isa CompiledScene + @test !bindings_dirty(cache_scene) + @test compiled_bindings(cache_scene) === cached_a + @test cached_a.revision == scene_revision(cache_scene) + @test refresh_bindings!(cache_scene) === cached_a + + register_object!(cache_scene, Object(:leaf_4; scale=:Leaf, kind=:plant, species=:oil_palm); parent=:plant_2) + @test bindings_dirty(cache_scene) + @test isnothing(compiled_bindings(cache_scene)) + cached_b = refresh_bindings!(cache_scene) + @test cached_b !== cached_a + @test cached_b.revision == scene_revision(cache_scene) + @test only(row for row in explain_scene_applications(cached_b) if row.application_id == :leaf_energy).target_ids == + [:leaf_1, :leaf_2, :leaf_3, :leaf_4] + @test only(row for row in explain_bindings(cached_b) if row.consumer_id == :leaf_3).source_ids == + [:leaf_3, :leaf_4] + + move_object!(cache_scene, :leaf_4, (x=3.0, y=0.0)) + @test !bindings_dirty(cache_scene) + @test environment_bindings_dirty(cache_scene) + @test refresh_bindings!(cache_scene) === cached_b + mark_environment_binding_dirty!(cache_scene) + @test !bindings_dirty(cache_scene) + + reparent_object!(cache_scene, :leaf_4, :plant_1) + cached_c = refresh_bindings!(cache_scene) + @test only(row for row in explain_bindings(cached_c) if row.consumer_id == :leaf_4).source_ids == + [:leaf_1, :leaf_2, :leaf_4] + + remove_object!(cache_scene, :leaf_4) + cached_d = refresh_bindings!(cache_scene) + @test only(row for row in explain_scene_applications(cached_d) if row.application_id == :leaf_energy).target_ids == + [:leaf_1, :leaf_2, :leaf_3] + + grid_backend = SceneObjectGridBackend(Any[]) + environment_specs = ( + ModelSpec(SceneObjectEnvironmentProbeModel(); name=:probe) |> + AppliesTo(Many(scale=:Leaf)) |> + Environment(provider=:grid), + ModelSpec(SceneObjectEnvironmentUpdateModel(); name=:temperature_update) |> + AppliesTo(Many(scale=:Leaf)) |> + Environment(provider=:grid), + ) + environment_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, species=:oil_palm, name=:palm_1, parent=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1, geometry=(cell=:cell_a,)), + Object(:leaf_2; scale=:Leaf, kind=:plant, species=:oil_palm, parent=:plant_1, geometry=(cell=:cell_b,)); + applications=environment_specs, + environment=grid_backend, + ) + compiled_environment = refresh_environment_bindings!(environment_scene) + @test compiled_environment isa CompiledEnvironmentBindings + @test !environment_bindings_dirty(environment_scene) + @test compiled_environment_bindings(environment_scene) === compiled_environment + @test length(compiled_environment.by_target) == length(compiled_environment.bindings) + @test compiled_environment.by_target[(:probe, ObjectId(:leaf_1))].cell == :cell_a + @test compiled_environment.by_target[(:temperature_update, ObjectId(:leaf_2))].cell == :cell_b + @test length(grid_backend.index_updates) == 1 + @test any(entity -> entity.id == :leaf_1 && entity.geometry == (cell=:cell_a,), grid_backend.index_updates[1]) + @test any(entity -> entity.id == :plant_1 && entity.scale == :Plant, grid_backend.index_updates[1]) + environment_rows = explain_environment_bindings(compiled_environment) + @test length(environment_rows) == 4 + leaf_1_probe = only(row for row in environment_rows if row.application_id == :probe && row.object_id == :leaf_1) + @test leaf_1_probe.provider == :grid + @test leaf_1_probe.cell == :cell_a + @test leaf_1_probe.required_inputs == [:T, :CO2] + @test leaf_1_probe.produced_outputs == Symbol[] + leaf_2_update = only(row for row in environment_rows if row.application_id == :temperature_update && row.object_id == :leaf_2) + @test leaf_2_update.cell == :cell_b + @test leaf_2_update.required_inputs == [:T] + @test leaf_2_update.produced_outputs == [:T] + + structural_environment_cache = refresh_bindings!(environment_scene) + move_object!(environment_scene, :leaf_2, (cell=:cell_c,)) + @test !bindings_dirty(environment_scene) + @test environment_bindings_dirty(environment_scene) + @test refresh_bindings!(environment_scene) === structural_environment_cache + refreshed_environment = refresh_environment_bindings!(environment_scene) + @test !environment_bindings_dirty(environment_scene) + @test length(grid_backend.index_updates) == 2 + @test any(entity -> entity.id == :leaf_2 && entity.geometry == (cell=:cell_c,), grid_backend.index_updates[2]) + @test only(row for row in explain_environment_bindings(refreshed_environment) if row.application_id == :probe && row.object_id == :leaf_2).cell == :cell_c + + update_geometry!(environment_scene, :leaf_1, (cell=:cell_e,); invalidate_environment=false) + @test geometry(only(object for object in scene_objects(environment_scene; scale=:Leaf) if object.id == ObjectId(:leaf_1))) == (cell=:cell_e,) + @test !environment_bindings_dirty(environment_scene) + mark_environment_binding_dirty!(environment_scene, :leaf_1) + @test environment_bindings_dirty(environment_scene) + refreshed_after_mark = refresh_environment_bindings!(environment_scene) + @test !environment_bindings_dirty(environment_scene) + @test length(grid_backend.index_updates) == 3 + @test any(entity -> entity.id == :leaf_1 && entity.geometry == (cell=:cell_e,), grid_backend.index_updates[3]) + @test only(row for row in explain_environment_bindings(refreshed_after_mark) if row.application_id == :probe && row.object_id == :leaf_1).cell == :cell_e + + register_object!(environment_scene, Object(:leaf_3; scale=:Leaf, kind=:plant, species=:oil_palm, geometry=(cell=:cell_d,)); parent=:plant_1) + @test bindings_dirty(environment_scene) + @test environment_bindings_dirty(environment_scene) + refreshed_with_new_leaf = refresh_environment_bindings!(environment_scene) + @test length(grid_backend.index_updates) == 4 + @test any(entity -> entity.id == :leaf_3 && entity.geometry == (cell=:cell_d,), grid_backend.index_updates[4]) + @test only(row for row in explain_scene_applications(refresh_bindings!(environment_scene)) if row.application_id == :probe).target_ids == + [:leaf_1, :leaf_2, :leaf_3] + @test only(row for row in explain_environment_bindings(refreshed_with_new_leaf) if row.application_id == :probe && row.object_id == :leaf_3).cell == :cell_d + + mutable_environment_backend = SceneObjectMutableEnvironmentBackend(:cell_a => 20.0, :cell_b => 30.0) + mutable_environment_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, geometry=(cell=:cell_a,), status=Status(T=0.0, temperature_update=0.0, temperature_seen=0.0)), + Object(:leaf_2; scale=:Leaf, kind=:plant, parent=:scene, geometry=(cell=:cell_b,), status=Status(T=0.0, temperature_update=0.0, temperature_seen=0.0)); + applications=( + ModelSpec(SceneObjectEnvironmentUpdateModel(); name=:temperature_update_runtime) |> + AppliesTo(Many(scale=:Leaf)) |> + Environment(provider=:grid), + ModelSpec(SceneObjectEnvironmentProbeModel(); name=:probe_after_update) |> + AppliesTo(Many(scale=:Leaf)) |> + Environment(provider=:grid), + ), + environment=mutable_environment_backend, + ) + run!(mutable_environment_scene) + @test mutable_environment_backend.values == Dict(:cell_a => 21.0, :cell_b => 31.0) + @test mutable_environment_backend.writes == [ + (application=:temperature_update_runtime, process=:scene_object_environment_update, cell=:cell_a, variable=:T, value=21.0, time=1), + (application=:temperature_update_runtime, process=:scene_object_environment_update, cell=:cell_b, variable=:T, value=31.0, time=1), + ] + mutable_environment_statuses = Dict(object.id.value => object.status for object in scene_objects(mutable_environment_scene; scale=:Leaf)) + @test mutable_environment_statuses[:leaf_1].temperature_seen == 21.0 + @test mutable_environment_statuses[:leaf_2].temperature_seen == 31.0 + + runtime_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:plant_1; scale=:Plant, kind=:plant, parent=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:plant_1, status=Status(leaf_area=1.5, leaf_areas=[0.0], leaf_tokens=Any[], carrier_total=0.0, temperature_seen=0.0)), + Object(:leaf_2; scale=:Leaf, kind=:plant, parent=:plant_1, status=Status(leaf_area=2.5, leaf_areas=[0.0], leaf_tokens=Any[], carrier_total=0.0, temperature_seen=0.0)); + applications=( + ModelSpec(SceneObjectCarrierConsumerModel(); name=:carrier_runtime) |> + AppliesTo(Many(scale=:Leaf)) |> + Inputs(:leaf_areas => Many(scale=:Leaf, within=SelfPlant(), var=:leaf_area)), + ModelSpec(SceneObjectEnvironmentProbeModel(); name=:probe_runtime) |> + AppliesTo(Many(scale=:Leaf)) |> + Environment(provider=:global), + ), + environment=(T=27.5, CO2=410.0), + ) + run!(runtime_scene) + @test all(object.status.carrier_total == 4.0 for object in scene_objects(runtime_scene; scale=:Leaf)) + @test all(object.status.temperature_seen == 27.5 for object in scene_objects(runtime_scene; scale=:Leaf)) + + call_runtime_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0, called_signal=0.0)); + applications=( + ModelSpec(SceneObjectSignalSourceModel(); name=:signal_source) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalCallerModel(); name=:signal_caller) |> + AppliesTo(One(scale=:Leaf)), + ), + ) + run!(call_runtime_scene) + call_status = only(scene_objects(call_runtime_scene; scale=:Leaf)).status + @test call_status.signal == 1.0 + @test call_status.called_signal == 1.0 + call_schedule = explain_schedule(refresh_bindings!(call_runtime_scene)) + @test only(row for row in call_schedule if row.application_id == :signal_source).manual_call_only + @test !only(row for row in call_schedule if row.application_id == :signal_source).root_scheduled + @test only(row for row in call_schedule if row.application_id == :signal_caller).root_scheduled + + hard_call_order_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0, called_signal=0.0, observed_signal=0.0)); + applications=( + ModelSpec(SceneObjectSignalConsumerModel(); name=:signal_consumer) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalSourceModel(); name=:signal_source) |> + AppliesTo(One(scale=:Leaf)), + ModelSpec(SceneObjectSignalCallerModel(); name=:signal_caller) |> + AppliesTo(One(scale=:Leaf)), + ), + ) + hard_call_order = refresh_bindings!(hard_call_order_scene) + @test hard_call_order.applications_by_id[:signal_caller].process == :scene_object_signal_caller + @test hard_call_order.application_order == [:signal_source, :signal_caller, :signal_consumer] + run!(hard_call_order_scene) + hard_call_order_status = only(scene_objects(hard_call_order_scene; scale=:Leaf)).status + @test hard_call_order_status.signal == 1.0 + @test hard_call_order_status.observed_signal == 1.0 + + temporal_input_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene, status=Status(signal_sum=0.0, temporal_total=0.0)), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0)); + applications=( + ModelSpec(SceneObjectSignalSourceModel(); name=:hourly_signal) |> + AppliesTo(One(scale=:Leaf)) |> + TimeStep(Hour(1)), + ModelSpec(SceneObjectTemporalSumModel(); name=:scene_temporal_sum) |> + AppliesTo(One(scale=:Scene)) |> + Inputs(:signal_sum => One(scale=:Leaf, var=:signal, policy=Integrate(), window=Hour(2))) |> + TimeStep(Hour(2)), + ), + environment=(duration=Hour(1),), + ) + temporal_binding = only( + row for row in explain_bindings(refresh_bindings!(temporal_input_scene)) + if row.application_id == :scene_temporal_sum && row.input == :signal_sum + ) + @test temporal_binding.carrier_hint == :temporal_stream + run!(temporal_input_scene; steps=3) + @test only(scene_objects(temporal_input_scene; scale=:Leaf)).status.signal == 3.0 + @test only(scene_objects(temporal_input_scene; scale=:Scene)).status.temporal_total == 5.0 + + temporal_holdlast_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene, status=Status(signal_sum=0.0, temporal_total=0.0)), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0)); + applications=( + ModelSpec(SceneObjectSignalSourceModel(); name=:hourly_signal) |> + AppliesTo(One(scale=:Leaf)) |> + TimeStep(Hour(1)), + ModelSpec(SceneObjectTemporalSumModel(); name=:scene_temporal_latest) |> + AppliesTo(One(scale=:Scene)) |> + Inputs(:signal_sum => One(scale=:Leaf, var=:signal, policy=HoldLast(), window=Hour(2))) |> + TimeStep(Hour(2)), + ), + environment=(duration=Hour(1),), + ) + run!(temporal_holdlast_scene; steps=3) + @test only(scene_objects(temporal_holdlast_scene; scale=:Scene)).status.temporal_total == 3.0 + + writer_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(biomass=-1.0)), + ) + biomass_source = + ModelSpec(SceneObjectBiomassSourceModel(); name=:carbon_allocation) |> + AppliesTo(One(scale=:Leaf)) + biomass_pruner = + ModelSpec(SceneObjectBiomassPrunerModel(); name=:leaf_pruning) |> + AppliesTo(One(scale=:Leaf)) + + @test_throws ErrorException compile_scene(writer_scene, (biomass_source, biomass_pruner)) + @test_throws ErrorException compile_scene( + writer_scene, + (biomass_source, biomass_pruner |> Updates(:biomass; after=:water_status)), + ) + @test_throws ErrorException compile_scene( + writer_scene, + (biomass_pruner |> Updates(:biomass; after=:carbon_allocation), biomass_source), + ) + + ordered_pruner = biomass_pruner |> Updates(:biomass; after=:carbon_allocation) + writer_compiled = compile_scene(writer_scene, (biomass_source, ordered_pruner)) + writer_row = only(row for row in explain_writers(writer_compiled) if row.variable == :biomass) + @test writer_row.object_id == :leaf_1 + @test writer_row.duplicate + @test writer_row.application_ids == [:carbon_allocation, :leaf_pruning] + @test writer_row.update_application_ids == [:leaf_pruning] + @test writer_row.update_after == [:leaf_pruning => [:carbon_allocation]] + + writer_runtime_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(biomass=-1.0)); + applications=(biomass_source, ordered_pruner), + ) + run!(writer_runtime_scene) + @test only(scene_objects(writer_runtime_scene; scale=:Leaf)).status.biomass == 0.0 + + multirate_scene = Scene( + Object(:scene; scale=:Scene, kind=:scene), + Object(:leaf_1; scale=:Leaf, kind=:plant, parent=:scene, status=Status(signal=0.0)); + applications=( + ModelSpec(SceneObjectSignalSourceModel(); name=:hourly_signal) |> + AppliesTo(One(scale=:Leaf)) |> + TimeStep(Hour(2)), + ), + environment=(duration=Hour(1),), + ) + multirate_compiled = refresh_bindings!(multirate_scene) + schedule_rows = explain_schedule(multirate_compiled) + @test only(schedule_rows).application_id == :hourly_signal + @test only(schedule_rows).dt_steps == 2.0 + @test only(schedule_rows).dt_seconds == 7200.0 + run!(multirate_scene; steps=5) + @test only(scene_objects(multirate_scene; scale=:Leaf)).status.signal == 3.0 +end diff --git a/test/test-updates.jl b/test/test-updates.jl new file mode 100644 index 000000000..082864926 --- /dev/null +++ b/test/test-updates.jl @@ -0,0 +1,86 @@ +using Dates + +PlantSimEngine.@process "update_carbon_allocation" verbose = false +PlantSimEngine.@process "update_leaf_pruning" verbose = false +PlantSimEngine.@process "update_leaf_senescence" verbose = false +PlantSimEngine.@process "update_biomass_observer" verbose = false + +struct UpdateCarbonAllocationModel <: AbstractUpdate_Carbon_AllocationModel end +PlantSimEngine.inputs_(::UpdateCarbonAllocationModel) = NamedTuple() +PlantSimEngine.outputs_(::UpdateCarbonAllocationModel) = (leaf_biomass=0.0,) +function PlantSimEngine.run!(::UpdateCarbonAllocationModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_biomass = 10.0 + return nothing +end + +struct UpdateLeafPruningModel <: AbstractUpdate_Leaf_PruningModel end +PlantSimEngine.inputs_(::UpdateLeafPruningModel) = NamedTuple() +PlantSimEngine.outputs_(::UpdateLeafPruningModel) = (leaf_biomass=0.0,) +function PlantSimEngine.run!(::UpdateLeafPruningModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_biomass = 0.0 + return nothing +end + +struct UpdateLeafSenescenceModel <: AbstractUpdate_Leaf_SenescenceModel end +PlantSimEngine.inputs_(::UpdateLeafSenescenceModel) = NamedTuple() +PlantSimEngine.outputs_(::UpdateLeafSenescenceModel) = (leaf_biomass=0.0,) +function PlantSimEngine.run!(::UpdateLeafSenescenceModel, models, status, meteo, constants=nothing, extra=nothing) + status.leaf_biomass *= 0.5 + return nothing +end + +struct UpdateBiomassObserverModel <: AbstractUpdate_Biomass_ObserverModel end +PlantSimEngine.inputs_(::UpdateBiomassObserverModel) = (leaf_biomass=0.0,) +PlantSimEngine.outputs_(::UpdateBiomassObserverModel) = (observed_biomass=0.0,) +function PlantSimEngine.run!(::UpdateBiomassObserverModel, models, status, meteo, constants=nothing, extra=nothing) + status.observed_biomass = status.leaf_biomass + return nothing +end + +@testset "ModelSpec Updates" begin + meteo = Atmosphere(T=20.0, Rh=0.65, Wind=1.0, duration=Dates.Hour(1)) + + @test_throws "Ambiguous canonical writers" ModelMapping( + UpdateCarbonAllocationModel(), + UpdateLeafPruningModel(), + status=(leaf_biomass=1.0,), + ) + + mapping = ModelMapping( + UpdateCarbonAllocationModel(), + ModelSpec(UpdateLeafPruningModel()) |> + Updates(:leaf_biomass; after=:update_carbon_allocation), + UpdateBiomassObserverModel(), + status=(leaf_biomass=1.0, observed_biomass=-1.0), + ) + + outputs = run!(mapping, meteo; executor=SequentialEx()) + @test only(outputs[:leaf_biomass]) == 0.0 + @test only(outputs[:observed_biomass]) == 0.0 + + graph_nodes = PlantSimEngine.traverse_dependency_graph(dep(mapping), false) + pruning_node = only(filter(node -> node.process == :update_leaf_pruning, graph_nodes)) + @test any(parent -> parent.process == :update_carbon_allocation, pruning_node.parent) + + @test_throws "without an ordering relation" ModelMapping( + UpdateCarbonAllocationModel(), + ModelSpec(UpdateLeafPruningModel()) |> + Updates(:leaf_biomass; after=:update_carbon_allocation), + ModelSpec(UpdateLeafSenescenceModel()) |> + Updates(:leaf_biomass; after=:update_carbon_allocation), + status=(leaf_biomass=1.0,), + ) + + ordered_updates = ModelMapping( + UpdateCarbonAllocationModel(), + ModelSpec(UpdateLeafSenescenceModel()) |> + Updates(:leaf_biomass; after=:update_carbon_allocation), + ModelSpec(UpdateLeafPruningModel()) |> + Updates(:leaf_biomass; after=(:update_carbon_allocation, :update_leaf_senescence)), + UpdateBiomassObserverModel(), + status=(leaf_biomass=1.0, observed_biomass=-1.0), + ) + ordered_outputs = run!(ordered_updates, meteo; executor=SequentialEx()) + @test only(ordered_outputs[:leaf_biomass]) == 0.0 + @test only(ordered_outputs[:observed_biomass]) == 0.0 +end