diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 364ddc81a..bc5bc8891 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,6 +40,8 @@ jobs: - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + env: + JULIA_NUM_THREADS: 4 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/Integration.yml b/.github/workflows/Integration.yml index 05f3d0a8a..4d0671497 100644 --- a/.github/workflows/Integration.yml +++ b/.github/workflows/Integration.yml @@ -32,8 +32,8 @@ jobs: arch: - x64 package: - #- {user: PalmStudio, repo: XPalm.jl} - - {user: VEZY, repo: PlantBioPhysics.jl} + - {user: PalmStudio, repo: XPalm.jl, branch: PSE-API-changes} + - {user: VEZY, repo: PlantBioPhysics.jl, branch: ModelList-outputs-filtering-changes} steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 @@ -45,6 +45,7 @@ jobs: uses: actions/checkout@v4 with: repository: ${{ matrix.package.user }}/${{ matrix.package.repo }} + ref: ${{matrix.package.branch}} path: downstream - name: Load this and run the downstream tests shell: julia --threads 4 --color=yes --project=downstream --depwarn=yes {0} diff --git a/.github/workflows/benchmarks_and_downstream.yml b/.github/workflows/benchmarks_and_downstream.yml new file mode 100644 index 000000000..0b0db73b3 --- /dev/null +++ b/.github/workflows/benchmarks_and_downstream.yml @@ -0,0 +1,70 @@ +name: BenchmarksAndDownstream +on: + push: + branches: + - main + - benchmarks-github-action + tags: "*" + workflow-dispatch: +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + env: + GROUP: ${{ matrix.package.group }} + strategy: + fail-fast: false + matrix: + version: + - "1" + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + package: + - {user: VEZY, repo: PlantSimEngine.jl, group: Downstream} + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + # TODO handle breaking changes the way downstream tests do ? + # NOTE : manifest toml file is removed otherwise git whines about untracked changes when switching branches for the gh-pages commit + - name: Run benchmarks + run: | + cd test/downstream + julia --project --threads 4 --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("test-all-benchmarks.jl")' + rm Manifest.toml + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Julia benchmark result + tool: 'julia' + output-file-path: ${{ github.workspace }}/test/downstream/output.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@Samuel-AMAP, @VEZY' + + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: false \ No newline at end of file diff --git a/README.md b/README.md index b12ccf89b..864091693 100644 --- a/README.md +++ b/README.md @@ -9,31 +9,54 @@ [![DOI](https://zenodo.org/badge/571659510.svg)](https://zenodo.org/badge/latestdoi/571659510) [![JOSS](https://joss.theoj.org/papers/137e3e6c2ddc349bec39e06bb04e4e09/status.svg)](https://joss.theoj.org/papers/137e3e6c2ddc349bec39e06bb04e4e09) +- [PlantSimEngine](#plantsimengine) + - [Overview](#overview) + - [Unique Features](#unique-features) + - [Automatic Model Coupling](#automatic-model-coupling) + - [Flexibility with Precision Control](#flexibility-with-precision-control) + - [Batteries included](#batteries-included) + - [Ask Questions](#ask-questions) + - [Installation](#installation) + - [Example usage](#example-usage) + - [Simple example](#simple-example) + - [Model coupling](#model-coupling) + - [Multiscale modelling](#multiscale-modelling) + - [Projects that use PlantSimEngine](#projects-that-use-plantsimengine) + - [Performance](#performance) + - [Make it yours](#make-it-yours) ## Overview -`PlantSimEngine` is a modelling framework for simulating and modelling plants, soil and atmosphere. It provides tools to **prototype, evaluate, test, and deploy** plant/crop models at any scale, with a strong emphasis on performance and efficiency. +`PlantSimEngine` is a comprehensive framework for building models of the soil-plant-atmosphere continuum. It includes everything you need to **prototype, evaluate, test, and deploy** plant/crop models at any scale, with a strong emphasis on performance and efficiency, so you can focus on building and refining your models. -**Key Features:** +**Why choose PlantSimEngine?** -- Process Definition: Easily define new processes such as light interception, photosynthesis, growth, soil water transfer, and more. -- Interactive Prototyping: Fast and interactive prototyping of models with built-in constraints to avoid errors and sensible defaults to streamline the model writing process. -- Control Degrees of Freedom: Fix variables, pass measurements, or use simpler models for specific processes to reduce complexity. -- Automatic Management: The package automatically manages input and output variables, time-steps, objects, and the coupling of models using a dependency graph. -- Flexible Model Switching: Switch between models without changing any code, using a simple syntax to specify the model for a given process. -- Integrated Data Use: Force variables to take measured values instead of model predictions, reducing degrees of freedom during model development and increasing accuracy during production mode. -- High-Performance Computation: Achieve high-speed computations, with benchmarks showing operations in the 100th of nanoseconds range for complex models (see this [benchmark script](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/benchmark.jl)). -- Parallel and Distributed Computing: Out-of-the-box support for sequential, multi-threaded, or distributed computations over objects, time-steps, and independent processes, thanks to [Floops.jl](https://juliafolds.github.io/FLoops.jl/stable/). -- Scalability: Scale easily with methods for computing over objects, time-steps, and [Multi-Scale Tree Graphs](https://github.com/VEZY/MultiScaleTreeGraph.jl). -- Composability: Use any types as inputs, including [Unitful](https://github.com/PainterQubits/Unitful.jl) for unit propagation and [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) for propagating measurement error. +- **Simplicity**: Write less code, focus on your model's logic, and let the framework handle the rest. +- **Modularity**: Each model component can be developed, tested, and improved independently. Assemble complex simulations by reusing pre-built, high-quality modules. +- **Standardisation**: Clear, enforceable guidelines ensure that all models adhere to best practices. This built-in consistency means that once you implement a model, it works seamlessly with others in the ecosystem. +- **Optimised Performance**: Don't re-invent the wheel. Delegating low-level tasks to PlantSimEngine guarantees that your model will benefit from every improvement in the framework. Enjoy faster prototyping, robust simulations, and efficient execution using Julia's high-performance capabilities. -**Benefits:** +## Unique Features -Improved Accuracy and Reliability: +### Automatic Model Coupling -- Enhance the accuracy of plant growth and yield predictions by integrating detailed physiological processes and environmental interactions. -- Reduced Modeling Time: Streamline the modeling process with automated management and fast prototyping capabilities. -- Collaborative Research: Facilitate collaborative research efforts with flexible and composable modeling tools. +**Seamless Integration:** PlantSimEngine leverages Julia's multiple-dispatch capabilities to automatically compute the dependency graph between models. This allows researchers to effortlessly couple models without writing complex connection code or manually managing dependencies. + +**Intuitive Multi-Scale Support:** The framework naturally handles models operating at different scales—from organelle to ecosystem—connecting them with minimal effort and maintaining consistency across scales. + +### Flexibility with Precision Control + +**Effortless Model Switching:** Researchers can switch between different component models using a simple syntax without rewriting the underlying model code. This enables rapid comparison between different hypotheses and model versions, accelerating the scientific discovery process. + +## Batteries included + +- **Automated Management**: Seamlessly handle inputs, outputs, time-steps, objects, and dependency resolution. +- **Iterative Development**: Fast and interactive prototyping of models with built-in constraints to avoid errors and sensible defaults to streamline the model writing process. +- **Control Your Degrees of Freedom**: Fix variables to constant values or force to observations, use simpler models for specific processes to reduce complexity. +- **High-Speed Computations**: Achieve impressive performance with benchmarks showing operations in the 100th of nanoseconds range for complex models (see this [benchmark script](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/benchmark.jl)). +- **Parallelize and Distribute Computing**: Out-of-the-box support for sequential, multi-threaded, or distributed computations over objects, time-steps, and independent processes, thanks to [Floops.jl](https://juliafolds.github.io/FLoops.jl/stable/). +- **Scale Effortlessly**: Methods for computing over objects, time-steps, and [Multi-Scale Tree Graphs](https://github.com/VEZY/MultiScaleTreeGraph.jl). +- **Compose Freely**: Use any types as inputs, including [Unitful](https://github.com/PainterQubits/Unitful.jl) for unit propagation and [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) for measurement error propagation. ## Ask Questions @@ -57,7 +80,7 @@ using PlantSimEngine The package is designed to be easy to use, and to help users avoid errors when implementing, coupling and simulating models. -### Simple example +### Simple example Here's a simple example of a model that simulates the growth of a plant, using a simple exponential growth model: @@ -193,47 +216,47 @@ fig ![LAI Growth and light interception](examples/LAI_growth2.png) -### Multiscale modelling +### Multiscale modelling -> See the [Multi-scale modeling](#multi-scale-modeling) section for more details. +> See the Multi-scale modeling section of the docs for more details. The package is designed to be easily scalable, and can be used to simulate models at different scales. For example, you can simulate a model at the leaf scale, and then couple it with models at any other scale, *e.g.* internode, plant, soil, scene scales. Here's an example of a simple model that simulates plant growth using sub-models operating at different scales: -```@example readme +```julia mapping = Dict( "Scene" => ToyDegreeDaysCumulModel(), "Plant" => ( MultiScaleModel( model=ToyLAIModel(), - mapping=[ + mapped_variables=[ :TT_cu => "Scene", ], ), Beer(0.6), MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil"], + mapped_variables=[:soil_water_content => "Soil"], ), MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] ], ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), MultiScaleModel( model=ToyInternodeEmergence(TT_emergence=20.0), - mapping=[:TT_cu => "Scene"], + mapped_variables=[:TT_cu => "Scene"], ), ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), Status(carbon_biomass=1.0) @@ -241,7 +264,7 @@ mapping = Dict( "Leaf" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), Status(carbon_biomass=1.0) @@ -254,13 +277,13 @@ mapping = Dict( We can import an example plant from the package: -```@example readme +```julia mtg = import_mtg_example() ``` Make a fake meteorological data: -```@example readme +```julia meteo = Weather( [ Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=300.0), @@ -271,7 +294,7 @@ meteo = Weather( And run the simulation: -```@example readme +```julia out_vars = Dict( "Scene" => (:TT_cu,), "Plant" => (:carbon_allocation, :carbon_assimilation, :soil_water_content, :aPPFD, :TT_cu, :LAI), @@ -285,9 +308,9 @@ out = run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()); We can then extract the outputs in a `DataFrame` and sort them: -```@example readme +```julia using DataFrames -df_out = outputs(out, DataFrame) +df_out = convert_outputs(out, DataFrame) sort!(df_out, [:timestep, :node]) ``` @@ -310,7 +333,6 @@ sort!(df_out, [:timestep, :node]) | 2 | Internode | 8 | 0.0627036 | | | | | | 0.75 | | 2 | Leaf | 9 | 0.0627036 | | | | | | 0.75 | - An example output of a multiscale simulation is shown in the documentation of PlantBiophysics.jl: ![Plant growth simulation](docs/src/www/image.png) @@ -319,11 +341,17 @@ An example output of a multiscale simulation is shown in the documentation of Pl Take a look at these projects that use PlantSimEngine: -- [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) -- [XPalm](https://github.com/PalmStudio/XPalm.jl) +- [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) - For the simulation of biophysical processes for plants such as photosynthesis, conductance, energy fluxes, and temperature +- [XPalm](https://github.com/PalmStudio/XPalm.jl) - An experimental crop model for oil palm + +## Performance + +PlantSimEngine delivers impressive performance for plant modeling tasks. On an M1 MacBook Pro, a toy model for leaf area over a year at daily time-scale took only 260 μs to perform (about 688 ns per day), and 275 μs (756 ns per day) when coupled to a light interception model. These benchmarks demonstrate performance on par with compiled languages like Fortran or C, far outpacing typical interpreted language implementations. + +For example, PlantBiophysics.jl, which implements ecophysiological models using PlantSimEngine, has been measured to run up to 38,000 times faster than equivalent implementations in other scientific computing languages. -## Make it yours +## Make it yours -The package is developed so anyone can easily implement plant/crop models, use it freely and as you want thanks to its MIT license. +The package is developed so anyone can easily implement plant/crop models, use it freely and as you want thanks to its MIT license. If you develop such tools and it is not on the list yet, please make a PR or contact me so we can add it! 😃 Make sure to read the community guidelines before in case you're not familiar with such things. diff --git a/docs/make.jl b/docs/make.jl index 9c82169c5..804f31cc0 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,3 +1,5 @@ +#using Pkg +#Pkg.develop("PlantSimEngine") using PlantSimEngine using PlantMeteo using DataFrames, CSV @@ -16,34 +18,66 @@ makedocs(; canonical="https://VirtualPlantLab.github.io/PlantSimEngine.jl", edit_link="main", assets=String[], - size_threshold=300000 - ), - pages=[ + size_threshold=500000 + ), pages=[ "Home" => "index.md", - "Design" => "design.md", - "Model Switching" => "model_switching.md", - "Reducing DoF" => "reducing_dof.md", + "Introduction" => [ + "Why PlantSimEngine ?" => "./introduction/why_plantsimengine.md", + "Why Julia ?" => "./introduction/why_julia.md", + ], + "Prerequisites" => [ + "Installing and running PlantSimEngine" => "./prerequisites/installing_plantsimengine.md", + "Key Concepts" => "./prerequisites/key_concepts.md", + "Julia language basics" => "./prerequisites/julia_basics.md", + ], + "Step by step - Single-scale simulations" => [ + "Detailed first simulation" => "./step_by_step/detailed_first_example.md", + "Coupling" => "./step_by_step/simple_model_coupling.md", + "Model Switching" => "./step_by_step/model_switching.md", + "Quick examples" => "./step_by_step/quick_and_dirty_examples.md", + "Implementing a process" => "./step_by_step/implement_a_process.md", + "Implementing a model" => "./step_by_step/implement_a_model.md", + "Parallelization" => "./step_by_step/parallelization.md", + "Advanced coupling and hard dependencies" => "./step_by_step/advanced_coupling.md", + "Implementing a model : additional notes" => "./step_by_step/implement_a_model_additional.md", + ], "Execution" => "model_execution.md", - "Fitting" => "fitting.md", - "Extending" => [ - "Processes" => "./extending/implement_a_process.md", - "Models" => "./extending/implement_a_model.md", - "Input types" => "./extending/inputs.md", + "Working with data" => [ + "Reducing DoF" => "./working_with_data/reducing_dof.md", + "Fitting" => "./working_with_data/fitting.md", + "Input types" => "./working_with_data/inputs.md", + "Visualizing outputs and data" => "./working_with_data/visualising_outputs.md", + "Floating-point considerations" => "./working_with_data/floating_point_accumulation_error.md", ], - "Coupling" => [ - "Users" => [ - "Simple case" => "./model_coupling/model_coupling_user.md", - "Multi-scale modelling" => "./model_coupling/multiscale.md", + "Moving to multiscale" => [ + "Multiscale considerations" => "./multiscale/multiscale_considerations.md", + "Converting a simulation to multi-scale" => "./multiscale/single_to_multiscale.md", + "More variable mapping examples" => "./multiscale/multiscale.md", + "Handling cyclic dependencies" => "./multiscale/multiscale_cyclic.md", + "Multiscale coupling considerations" => "./multiscale/multiscale_coupling.md", + "Building a simple plant" => [ + "A rudimentary plant simulation" => "./multiscale/multiscale_example_1.md", + "Expanding the plant simulation" => "./multiscale/multiscale_example_2.md", + "Fixing bugs in the plant simulation"=> "./multiscale/multiscale_example_3.md", ], - "Modelers" => "./model_coupling/model_coupling_modeler.md", - "Tips and Workarounds" => "./model_coupling/tips_and_workarounds.md", - ], - "FAQ" => ["./FAQ/translate_a_model.md"], - "API" => "API.md", + "Visualizing our toy plant with PlantGeom"=> "./multiscale/multiscale_example_4.md", + ], "Troubleshooting and testing" => [ + "Troubleshooting" => "./troubleshooting_and_testing/plantsimengine_and_julia_troubleshooting.md", + "Automated testing" => "./troubleshooting_and_testing/downstream_tests.md", + "Tips and Workarounds" => "./troubleshooting_and_testing/tips_and_workarounds.md", + "Implicit contracts" => "./troubleshooting_and_testing/implicit_contracts.md", + ], "API" => [ + "Public API" => "./API/API_public.md", + "Example models" => "./API/API_examples.md", + "Internal API" => "./API/API_private.md",], + "Improving our documentation" => "documentation_improvement.md", + "Developer guidelines" => "developers.md", + "Planned features" => "planned_features.md", ] ) deploydocs(; repo="github.com/VirtualPlantLab/PlantSimEngine.jl.git", - devbranch="main" + devbranch="main", + push_preview=true, # Visit https://VirtualPlantLab.github.io/PlantSimEngine.jl/previews/PR128 to visualize the preview of the PR #128 ) diff --git a/docs/src/API.md b/docs/src/API.md deleted file mode 100644 index 26ffca2bf..000000000 --- a/docs/src/API.md +++ /dev/null @@ -1,40 +0,0 @@ -# API - -## Index - -```@index -Modules = [PlantSimEngine] -``` - -## API documentation - -```@autodocs -Modules = [PlantSimEngine] -Private = false -``` - -## Un-exported - -Private functions, types or constants from `PlantSimEngine`. These are not exported, so you need to use `PlantSimEngine.` to access them (*e.g.* `PlantSimEngine.DataFormat`). - -```@autodocs -Modules = [PlantSimEngine] -Public = false -Private = true -``` - -## Example models - -PlantSimEngine provides example processes and models to users. They are available from a sub-module called `Examples`. To get access to these models, you can simply use this sub-module: - -```julia -using PlantSimEngine.Examples -``` - -The models are detailed below. - -```@autodocs -Modules = [PlantSimEngine.Examples] -Public = true -Private = true -``` diff --git a/docs/src/API/API_examples.md b/docs/src/API/API_examples.md new file mode 100644 index 000000000..e9e989266 --- /dev/null +++ b/docs/src/API/API_examples.md @@ -0,0 +1,21 @@ +# Example models + +PlantSimEngine provides example processes and models to users. They are available from a sub-module called `Examples`. To get access to these models in a working environment with PlantSimEngine, you can simply use this sub-module: + +```julia +using PlantSimEngine.Examples +``` + +## List + +```@index +Pages = ["API_examples.md"] +``` + +## Details + +```@autodocs +Modules = [PlantSimEngine.Examples] +Public = true +Private = true +``` \ No newline at end of file diff --git a/docs/src/API/API_private.md b/docs/src/API/API_private.md new file mode 100644 index 000000000..40b584bd8 --- /dev/null +++ b/docs/src/API/API_private.md @@ -0,0 +1,18 @@ +# API - internal functions +## Un-exported + +Private functions, types or constants from `PlantSimEngine`. These are not exported, so you need to use `PlantSimEngine.` to access them (*e.g.* `PlantSimEngine.DataFormat`). Most of them are developer code, but some may be useful for tinkerers, or to have greater control over some simulation parameters (future versions of this documentation might break those categories into separate pages for clarity). + +## Index + +```@index +Pages = ["API_private.md"] +``` + +## API documentation + +```@autodocs +Modules = [PlantSimEngine] +Public = false +Private = true +``` diff --git a/docs/src/API/API_public.md b/docs/src/API/API_public.md new file mode 100644 index 000000000..6e52bd2e4 --- /dev/null +++ b/docs/src/API/API_public.md @@ -0,0 +1,14 @@ +# Public API + +## Index + +```@index +Pages = ["API_public.md"] +``` + +## API documentation + +```@autodocs +Modules = [PlantSimEngine] +Private = false +``` diff --git a/docs/src/FAQ/translate_a_model.md b/docs/src/FAQ/translate_a_model.md index ebea01d07..92f18fed8 100644 --- a/docs/src/FAQ/translate_a_model.md +++ b/docs/src/FAQ/translate_a_model.md @@ -16,8 +16,6 @@ function lai_toymodel(TT_cu; max_lai=8.0, dd_incslope=500, inc_slope=70, dd_decs end meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) -# Note: meteo_day is defined below if you want to reproduce it, then use this to write it: -# PlantMeteo.write_weather("examples/meteo_day.csv", meteo_day, duration = Dates.Day) ``` If you already have a model, you can easily use `PlantSimEngine` to couple it with other models with minor adjustments. @@ -64,7 +62,7 @@ The model can be implemented using `PlantSimEngine` as follows: #### Define a process -If the process of LAI dynamic is not implement yet, we can define it like so: +If the process of LAI dynamic is not implemented yet, we can define it like so: ```julia @process LAI_Dynamic @@ -153,7 +151,7 @@ m = ModelList( status = (TT_cu = cumsum(meteo_day.TT),), ) -run!(m) +outputs_sim = run!(m) -lines(m[:TT_cu], m[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Days since sowing")) +lines(outputs_sim[:TT_cu], outputs_sim[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Days since sowing")) ``` diff --git a/docs/src/design.md b/docs/src/design.md deleted file mode 100644 index d6037c03f..000000000 --- a/docs/src/design.md +++ /dev/null @@ -1,246 +0,0 @@ -# Package design - -`PlantSimEngine.jl` is designed to ease the process of modelling and simulation of plants, soil and atmosphere, or really any system (*e.g.* agroforestry system, agrivoltaics...). `PlantSimEngine.jl` aims at being the backbone tool for developing Functional-Structural Plant Models (FSPM) and crop models without the hassle of performance and other computer-science considerations. - -```@setup usepkg -using PlantSimEngine, PlantMeteo -using PlantSimEngine.Examples -meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) -leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) -run!(leaf, meteo) -``` - -## Definitions - -### Processes - -A process in this package defines a biological or physical phenomena. Think of any process happening in a system, such as light interception, photosynthesis, water, carbon and energy fluxes, growth, yield or even electricity produced by solar panels. - -A process is "declared", meaning we just define a process using [`@process`](@ref), and then we implement models for its simulation. Declaring a process generates some boilerplate code for its simulation: - -- an abstract type for the process -- a method for the `process` function, that is used internally - -For example, the `light_interception` process is declared using: - -```julia -@process light_interception -``` - -Which would generate a tutorial to help the user implement a model for the process. - -The abstract process type is then used as a supertype of all models implementations for the process, and is named `AbstractProcess`, *e.g.* `AbstractLight_InterceptionModel`. - -### Models (ModelList) - -A process is simulated using a particular implementation, or **a model**. Each model is implemented using a structure that lists the parameters of the model. For example, PlantBiophysics provides the [`Beer`](https://vezy.github.io/PlantBiophysics.jl/stable/functions/#PlantBiophysics.Beer) structure for the implementation of the Beer-Lambert law of light extinction. The process of `light_interception` and the `Beer` model are provided as an example -script in this package too at [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/Beer.jl). - -Models can use three types of entries: - -- Parameters -- Meteorological information -- Variables -- Constants -- Extras - -Parameters are constant values that are used by the model to compute its outputs. Meteorological information are values that are provided by the user and are used as inputs to the model. It is defined for one time-step, and `PlantSimEngine.jl` takes care of applying the model to each time-steps given by the user. Variables are either used or computed by the model and can optionally be initialized before the simulation. Constants are constant values, usually common between models, *e.g.* the universal gas constant. And extras are just extra values that can be used by a model, it is for example used to pass the current node of the Multi-Scale Tree Graph to be able to *e.g.* retrieve children or ancestors values. - -Users can choose which model is used to simulate a process using the [`ModelList`](@ref) structure. `ModelList` is also used to store the values of the parameters, and to initialize variables. - -For example let's instantiate a [`ModelList`](@ref) with the Beer-Lambert model of light extinction. The model is implemented with the [`Beer`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/Beer.jl) structure and has only one parameter: the extinction coefficient (`k`). - -Importing the package: - -```@example usepkg -using PlantSimEngine -``` - -Import the examples defined in the `Examples` sub-module (`light_interception` and `Beer`): - -```julia -using PlantSimEngine.Examples -``` - -And then making a [`ModelList`](@ref) with the `Beer` model: - -```@example usepkg -ModelList(Beer(0.5)) -``` - -What happened here? We provided an instance of the `Beer` model to a [`ModelList`](@ref) to simulate the light interception process. - -## Parameters - -A parameter is a constant value that is used by a model to compute its outputs. For example, the Beer-Lambert model uses the extinction coefficient (`k`) to compute the light extinction. The Beer-Lambert model is implemented with the `Beer` structure, which has only one field: `k`. We can see that using `fieldnames` on the model structure: - -```@example usepkg -fieldnames(Beer) -``` - -## Variables (inputs, outputs) - -Variables are either input or outputs (*i.e.* computed) of models. Variables and their values are stored in the [`ModelList`](@ref) structure, and are initialized automatically or manually. - -Hence, [`ModelList`](@ref) objects stores two fields: - -```@example usepkg -fieldnames(ModelList) -``` - -The first field is a list of models associated to the processes they simulate. The second, `:status`, is used to hold all inputs and outputs of our models, called variables. For example the `Beer` model needs the leaf area index (`LAI`, m² m⁻²) to run. - -We can see which variables are needed as inputs using [`inputs`](@ref): - -```@example usepkg -inputs(Beer(0.5)) -``` - -and the outputs of the model using [`outputs`](@ref): - -```@example usepkg -outputs(Beer(0.5)) -``` - -If we instantiate a [`ModelList`](@ref) with the Beer-Lambert model, we can see that the `:status` field has two variables: `LAI` and `PPFD`. The first is an input, the second an output (*i.e.* it is computed by the model). - -```@example usepkg -m = ModelList(Beer(0.5)) -keys(status(m)) -``` - -To know which variables should be initialized, we can use [`to_initialize`](@ref): - -```@example usepkg -m = ModelList(Beer(0.5)) -to_initialize(m) -``` - -Their values are uninitialized though (hence the warnings): - -```@example usepkg -(m[:LAI], m[:aPPFD]) -``` - -Uninitialized variables are initialized to the value given in the `inputs` or `outputs` methods, which is usually equal to `typemin()`, *e.g.* `-Inf` for `Float64`. - -!!! tip - Prefer using `to_initialize` rather than `inputs` to check which variables should be initialized. `inputs` returns the variables that are needed by the model to run, but `to_initialize` returns the variables that are needed by the model to run and that are not initialized in the `ModelList`. Also `to_initialize` is more clever when coupling models (see below). - -We can initialize the variables by providing their values to the status at instantiation: - -```@example usepkg -m = ModelList(Beer(0.5), status = (LAI = 2.0,)) -``` - -Or after instantiation using [`init_status!`](@ref): - -```@example usepkg -m = ModelList(Beer(0.5)) - -init_status!(m, LAI = 2.0) -``` - -We can check if a component is correctly initialized using [`is_initialized`](@ref): - -```@example usepkg -is_initialized(m) -``` - -Some variables are inputs of models, but outputs of other models. When we couple models, `PlantSimEngine.jl` is clever and only requests the variables that are not computed by other models. - -## Climate forcing - -To make a simulation, we usually need the climatic/meteorological conditions measured close to the object or component. - -Users are strongly encouraged to use [`PlantMeteo.jl`](https://github.com/PalmStudio/PlantMeteo.jl), the companion package that helps manage such data, with default pre-computations and structures for efficient computations. The most basic data structure from this package is a type called [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere), which defines steady-state atmospheric conditions, *i.e.* the conditions are considered at equilibrium. Another structure is available to define different consecutive time-steps: [`TimeStepTable`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.TimeStepTable). - -The mandatory variables to provide for an [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere) are: `T` (air temperature in °C), `Rh` (relative humidity, 0-1) and `Wind` (the wind speed, m s⁻¹). In our example, we also need the incoming photosynthetically active radiation flux (`Ri_PAR_f`, W m⁻²). We can declare such conditions like so: - -```@example usepkg -using PlantMeteo -meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) -``` - -More details are available from the [package documentation](https://vezy.github.io/PlantMeteo.jl/stable). - -## Simulation - -### Simulation of processes - -Making a simulation is rather simple, we simply use [`run!`](@ref) on the `ModelList`: - -The call to [`run!`](@ref) is the same whatever the models you choose for simulating the processes. This is some magic allowed by `PlantSimEngine.jl`! Here is an example: - -```julia -run!(model_list, meteo) -``` - -The first argument is the model list (see [`ModelList`](@ref)), and the second defines the micro-climatic conditions. - -The `ModelList` should be initialized for the given process before calling the function. See [Variables (inputs, outputs)](@ref) for more details. - -### Example simulation - -For example we can simulate the `light_interception` of a leaf like so: - -```@example usepkg -using PlantSimEngine, PlantMeteo - -# Import the examples defined in the `Examples` sub-module -using PlantSimEngine.Examples - -meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) - -leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) - -run!(leaf, meteo) - -leaf[:aPPFD] -``` - -### Outputs - -The `status` field of a [`ModelList`](@ref) is used to initialize the variables before simulation and then to keep track of their values during and after the simulation. We can extract the simulation outputs of a model list using the [`status`](@ref) function. - -The status is usually stored in a `TimeStepTable` structure from `PlantMeteo.jl`, which is a fast DataFrame-alike structure with each time step being a [`Status`](@ref). It can be also be any `Tables.jl` structure, such as a regular `DataFrame`. The weather is also usually stored in a `TimeStepTable` but with each time step being an `Atmosphere`. - -Let's look at the status of our previous simulated leaf: - -```@setup usepkg -status(leaf) -``` - -We can extract the value of one variable using the `status` function, *e.g.* for the intercepted light: - -```@example usepkg -status(leaf, :aPPFD) -``` - -Or similarly using the dot syntax: - -```@example usepkg -leaf.status.aPPFD -``` - -Or much simpler (and recommended), by indexing directly into the model list: - -```@example usepkg -leaf[:aPPFD] -``` - -Another simple way to get the results is to transform the outputs into a `DataFrame`. Which is very easy because the `TimeStepTable` implements the Tables.jl interface: - -```@example usepkg -using DataFrames -DataFrame(leaf) -``` - -!!! note - The output from `DataFrame` is adapted to the kind of simulation you did: one row per time-step, and per component models if you simulated several. - -## Model coupling - -A model can work either independently or in conjunction with other models. For example a stomatal conductance model is often associated with a photosynthesis model, *i.e.* it is called from the photosynthesis model. - -`PlantSimEngine.jl` is designed to make model coupling painless for modelers and users. Please see [Model coupling for users](@ref) and [Model coupling for modelers](@ref) for more details. \ No newline at end of file diff --git a/docs/src/developers.md b/docs/src/developers.md new file mode 100644 index 000000000..b8ef8f2d1 --- /dev/null +++ b/docs/src/developers.md @@ -0,0 +1,125 @@ +# Developer guidelines + +This page is intended for people who wish to contribute to PlantSimEngine, and indicates the various parts to bear in mind when adding in new code. + +## Working on PlantSimEngine + +Instructions are no different than for any other package. Use git to clone the repository [https://github.com/VirtualPlantLab/PlantSimEngine.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl). + +When testing your changes, your environement will need to use a command such as `Pkg.develop("PlantSimEngine")` to make use of your code. + +We work with VSCode and are most comfortable with that IDE for Julia development. We mostly follow the manual's [Julia style guide](https://docs.julialang.org/en/v1/manual/style-guide/) + +Once you've made the necessary checks (see the [Checklist before submitting PRs](@ref) listed below), you’ll need to create your pull request and ask to be added to the contributors if you wish to submit new changes. + +This documentation has a [Roadmap](@ref). The list of known issues and related discussions can be found [here](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues). Some are outdated, some are discussions related to potential features, but others are genuine bugs or enhancement suggestions. + +Other details and questions can be posted on our issues page, or as part of your Pull Request. + +## Quick rundown + +### Testing environments + +PlantSimEngine has several developer environements: + +- `/PlantSimEngine/test`, to check for non-regressions +- `/PlantSimEngine/test/downstream`, whose folder contains a few benchmarks on PlantSimEngine, PlantBioPhysics and XPalm, run as a Github Action, to ensure changes don't cause performance regressions in packages depending on PlantSimEngine. You’ll need to have a version of those packages accessible if you wish to test them locally. Those are distinct from the Github Action that does some integration checks to ensure no unexpected breaking changes occurs. + `/PlantSimEngine/docs`, to build the documentation. The documentation runs code, and some of the functions' documentation for the API are also tested as `jldoctest` instances + +### Running the standard test suite + +Simply execute the `/PlantSimEngine/test/runtests.jl` file in the test environment. Note that you'll need to start Julia with multiple threads for the multi-threading tests to successfully run. + +You'll also need the companion packages PlantMeteo and MultiScaleTreeGraph, as well as other Julia packages such as DataFrames, CSV, Documenter, Test, Aqua and Tables. + +### Downstream tests + +With XPalm and PlantBioPhysics properly instantiated, execute the `/PlantSimEngine/test/downstream/test/test-all-benchmarks.jl`. You may need to add some packages for the script to run locally. + +### Building the documentation + +In the `/PlantSimEngine/docs` environment, run `/PlantSimEngine/docs/make.jl`. It requires a couple of packages that aren't compulsory elsewhere (Documenter, CairoMakie, PlantGeom). + +### Editing benchmarks + +⁃ If you wish for a branch to be benchmarked after every commit, then you need to declare it in the Github Action for benchmarks's yml file : [https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/.github/workflows/benchmarks_and_downstream.yml](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/.github/workflows/benchmarks_and_downstream.yml) and add your branch to the `on: push:` section. +⁃ You can view benchmarks here: . They are still somewhat WIP and not yet battle-tested. +⁃ You may occasionally need to update or delete a benchmark, in which case you will need to manually delete it in the **gh-pages** branch, in `dev/bench/index.html` +⁃ The actual benchmark list is located in the `test/downstream` folder. + +## Things to keep an eye out for + +### Check downstream tests + +⁃ If your changes affect the API, then they might affect a package depending on PlantSimEngine. Benchmarks can be a way to check, as some benchmarks run other packages. Otherwise, a specific GitHub action, [https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/.github/workflows/Integration.yml] runs other packages’ test suites. If this action fails, then it is likely some breaking change was introduced that hasn’t been accounted for in the downstream package. If you expected a breaking change and labelled your release as such, there will be no action failure +⁃ Note that those tests don’t build the doc (iirc), so they don’t cover that. +⁃ API changes can also affect downstream packages’ documentation and tests... + +### Which documentation pages may be affected by changes + +You may impact several specific documentation pages depending on what you changed. Features and API changes affect whatever they might affect, but there are some less obvious ramifications: + +⁃ Improving user errors may impact the **Troubleshooting** page. +⁃ Extra features might also expand the **Tips and workarounds** page, as well as the ‘implicit contracts’ page. +⁃ Some experimental features might be worth documenting in the dedicated **API** page, once it's added +⁃ The roadmap "**Planned features**" page needs updating +⁃ Potentially, other pages such as the **Credits** page, **Key Concepts**, etc. If the API makes use of new Julia features or syntax, the **Julia basics** page is probably also worth updating. +⁃ New examples are worth making doctests of. + +### Previewing documentation + +You can preview generated documentation (assuming it was able to build) relating to your PR (example given with #128) by checking the related link: [https://virtualplantlab.github.io/PlantSimEngine.jl/previews/PR128/](https://virtualplantlab.github.io/PlantSimEngine.jl/previews/PR128/) + +## Checklist before submitting PRs + +⁃ Ensure your code, uh, works +⁃ Ensure your major changes are covered by some tests, and new features are documented +⁃ Run the PlantSimEngine test suite locally and check errors +⁃ Check on Github which issues it affects, and update/comment those issues, or link them to your Pull Request +⁃ Check which doc page changes are needed (roadmap, … see further up), and update those +⁃ Build the PSE doc and update whatever doc tests were broken +⁃ Push your commit, and let the Github Actions run their course +⁃ Check the 'CI' GitHub action and fix if necessary +⁃ Check downstream and benchmark GitHub actions: + - If benchmarks tanked, then fix your code. If you need to add/update/delete benchmarks, do so. + - If you broke an integration/downstream test, you’ll need to investigate it + - If API changes were made, also check downstream packages’ documentation + +It’s probably now safe to request a merge. + +### A few extra things worth doing + +⁃ You may have some new known issues, some remaining TODOs, document those somewhere, whether in the PR comments or in their own issue, make sure some trace remains +⁃ Finally, update this page and this checklist: If a doc page is added, it may be part of the list of pages you need to keep an eye on. If proper memory allocation tracking and type stability checking is implemented, then that’ll need to be added to the list of things to check prior to a release, etc. + +### Other helpful things + +⁃ In the `/PlantSimEngine/test` folder, there are a few basic helper functions. One of them outputs vectors of modellists, weather data, and output variables, which are used as a test bank/matrix for some tests, and provides wide coverage. If you wrote new models, new combinations of models, or added some new weather data, it helps to add them to the banks. +⁃ New downstream packages are worth adding to the integration and downstream package registry. +⁃ Unusual corner-cases are worth giving their own unit tests. Newly fixed bugs as well, even if the fix is fairly trivial. + +## Noteworthy aspects of the codebase + +### Automatic model generation + +A specific feature requires generating models on the fly, to enable passing vectors to `Status` objects in multi-scale simulations. There may be more features that wish to generate models. + +The solution makes use of a somewhat brittle feature, `eval()`, with some subtleties. You can read more about the related world age problem [here](https://arxiv.org/abs/2010.07516), or [here](https://discourse.julialang.org/t/world-age-problem-explanation/9714/15). + +The related file is `model_generation_from_status_vectors.jl`, which has some additional comments. + +What is important to bear in mind, is that if you call functions which generate models via `eval()`, you will need to return to top-level scope for those changes to become visible. You can see an example in `tests/helper_functions.jl` with the functions `test_filtered_output_begin` and `test_filtered_output`. The first function calls `modellist_to_mapping`, which creates some models on the fly to convert status vectors between a ModelList and its equivalent pseudo-multiscale mapping. The function is split in two so that it is possible to return to global scope and make the `eval()` changes publicly available. The second function then is able to run the simulations on the mapping with its generated models, and complete the test successfully. + +The errors returned by an `eval()`-related issue are very specific, and indicate that a generated model with an UUID suffix does not exist in the Main module, or something along those lines. + +There may be a better approach that avoids those pitfalls, but that's what we have for now. Be cautious when calling functions from that file, and make sure to look out for comments indicating a function was split into two. + +### Weather/timestep/status combinations + +Not all combinations of weather data structure/weather dataset size/status sizes combinations are tested in PlantSimEngine itself. Some are tested in PlantBioPhysics and XPalm. It'd be good to have those structures tested in PSE in the future, but for now it is highly recommended checking those packages' tests when changing the API. + +### Test banks + +They were briefly mentioned earlier in the page, but the test banks to increase the number of combinations tested for in terms of weather data, modellists/mappings and tracked outputs, could definitely be improved upon. + +Some additional work and tests regarding tracking memory allocations, type stability etc. would also be worth implementing/documenting. diff --git a/docs/src/documentation_improvement.md b/docs/src/documentation_improvement.md new file mode 100644 index 000000000..ef472f6ce --- /dev/null +++ b/docs/src/documentation_improvement.md @@ -0,0 +1,7 @@ +# Help improve our documentation ! + +One goal for PlantSimEngine is to ensure testing ecophysiological hypotheses, or building plant simulations is as easy as can be for a wider range of people than previous frameworks. + +Good documentation is essential for that purpose. + +If parts of the documentation are unclear to you, you are very welcome to send a PR, an email, or a message (either [on Github](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues) or [on the FSPM Discourse](https://fspm.discourse.group/latest)) so that we can improve upon it. \ No newline at end of file diff --git a/docs/src/extending/implement_a_model.md b/docs/src/extending/implement_a_model.md deleted file mode 100644 index c9b13aa67..000000000 --- a/docs/src/extending/implement_a_model.md +++ /dev/null @@ -1,272 +0,0 @@ -# [Model implementation in 5 minutes](@id model_implementation_page) - -```@setup usepkg -using PlantSimEngine -@process "light_interception" verbose = false -struct Beer{T} <: AbstractLight_InterceptionModel - k::T -end -``` - -## Introduction - -`PlantSimEngine.jl` was designed to make new model implementation very simple. So let's learn about how to implement your own model with a simple example: implementing a new light interception model. - -## Inspiration - -If you want to implement a new model, the best way to do it is to start from another implementation. - -For a complete example, you can look at the code in [`PlantBiophysics.jl`](https://github.com/VEZY/PlantBiophysics.jl), were you will find *e.g.* a photosynthesis model, with the implementation of the `FvCB` model in this Julia file: [src/photosynthesis/FvCB.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/photosynthesis/FvCB.jl); an energy balance model with the implementation of the `Monteith` model in [src/energy/Monteith.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl); or a stomatal conductance model in [src/conductances/stomatal/medlyn.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/conductances/stomatal/medlyn.jl). - -`PlantSimEngine` also provide toy models that can be used as a base to better understand how to implement a new model: - -- The Beer model for light interception in [examples/Beer.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/Beer.jl) -- A toy LAI development in [examples/ToyLAIModel.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyLAIModel.jl) - -## Requirements - -In those files, you'll see that in order to implement a new model you'll need to implement: - -- a structure, used to hold the parameter values and to dispatch to the right method -- the actual model, developed as a method for the process it simulates -- some helper functions used by the package and/or the users - -If you create your own process, the function will print a short tutorial on how to do all that, adapted to the process you just created (see [Implement a new process](@ref)). - -In this page, we'll just implement a model for a process that already exists: the light interception. This process is defined in `PlantBiophysics.jl`, and also made available as an example model from the `Examples` sub-module. You can access the script from here: [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/Beer.jl). - -We can import the model like so: - -```julia -# Import the example models defined in the `Examples` sub-module: -using PlantSimEngine.Examples -``` - -But instead of just using it, we will review the script line by line. - -## Example: the Beer-Lambert model - -### The process - -We declare the light interception process at l.7 using [`@process`](@ref): - -```julia -@process "light_interception" verbose = false -``` - -See [Implement a new process](@ref) for more details on how that works and how to use the process. - -### The structure - -To implement a model, the first thing to do is to define a structure. The purpose of this structure is two-fold: - -- hold the parameter values -- dispatch to the right `run!` method when calling it - -The structure of the model (or type) is defined as follows: - -```@example usepkg -struct Beer{T} <: AbstractLight_InterceptionModel - k::T -end -``` - -The first line defines the name of the model (`Beer`), which is completely free, except it is good practice to use camel case for the name, *i.e.* using capital letters for the words and no separator `LikeThis`. - -We also can see that we define the `Beer` structure as a subtype of `AbstractLight_InterceptionModel`. This step is very important as it tells to the package what kind of process the model simulates. `AbstractLight_InterceptionModel` is automatically created when defining the process "light_interception". - -In our case, it tells us that `Beer` is a model to simulate the light interception process. - -Then comes the parameters names, and their types. The type of parameters is given by the user at instantiation in our example. This is done using the `T` notation as follows: - -- we say that our structure `Beer` is a parameterized `struct` by putting `T` in between brackets after the name of the `struct` -- We put `::T` after our parameter name in the `struct`. This way Julia knows that our parameter will be of type `T`. - -The `T` is completely free, you can use any other letter or word instead. If you have parameters that you know will be of different types, you can either force their type, or make them parameterizable too, using another letter, *e.g.*: - -```julia -struct YourStruct{T,S} <: AbstractLight_InterceptionModel - k::T - x::T - y::T - z::S -end -``` - -Parameterized types are very useful because they let the user choose the type of the parameters, and potentially dispatch on them. - -But why not forcing the type such as the following: - -```julia -struct YourStruct <: AbstractLight_InterceptionModel - k::Float64 - x::Float64 - y::Float64 - z::Int -end -``` - -Well, you can do that. But you'll lose a lot of the magic Julia has to offer this way. - -For example a user could use the `Particles` type from [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) to make automatic uncertainty propagation, and this is only possible if the type is parameterizable. - -### The method - -The models are implemented by adding a method for its type to the [`run!`](@ref) function. The exclamation point at the end of the function name is used in Julia to tell users that the function is mutating, *i.e.* it modifies its input. - -The function takes six arguments: - -- the type of your model -- models: a `ModelList` object, which contains all the models of the simulation -- status: a `Status` object, which contains the current values (*i.e.* state) of the variables for **one** time-step (e.g. the value of the plant LAI at time t) -- meteo: (usually) an `Atmosphere` object, or a row of the meteorological data, which contains the current values of the meteorological variables for **one** time-step (*e.g.* the value of the PAR at time t) -- constants: a `Constants` object, or a `NamedTuple`, which contains the values of the constants for the simulation (*e.g.* the value of the Stefan-Boltzmann constant) -- extras: any other object you want to pass to your model. This is for advanced users, and is not used in this example. Note that it is used to pass the `Node` when simulating a MultiScaleTreeGraph. - -Your implementation can use any variables or parameters in these objects. The only thing you have to do is to make sure that the variables you use are defined in the `Status` object, the meteorology, and the `Constants` object. - -The variables you use from the `Status` must be declared as inputs of your model. And the ones you modify must be declared as outputs. We'll that below. - -!!! warning - Models implementations are done for **one** time-step by design. The values of the previous time-step is always available in the `status` (*e.g.* `status.biomass`) as long as the variable is an output of your model. This is because at the end of a time-step, the `Status` object is recycled for the next time-step and so the latest computed values are always available. This is why it is possible to increment a value every time-step using *e.g.* `status.biomass += 1.0`. By design models don't have access to values prior to the one before. If you're not convinced by this approach, ask yourself how the plant knows the value of *e.g.* LAI from 15 days ago. It doesn't. It only knows its current state. Most of the time-sensitive variables really are just an accumulation of values until a threshold anyway. BUt if you really need to use values from the past (*e.g.* 15 time-steps before), you can add a variable to the `Status` object that is uses like a queue (see *e.g.* [DataStructures.jl](https://juliacollections.github.io/DataStructures.jl/stable/)). - -`PlantSimEngine` then automatically deals with every other detail, such as checking that the object is correctly initialized, applying the computations over objects and time-steps. This is nice because as a developer you don't have to deal with those details, and you can just concentrate on your model implementation. - -!!! warning - You need to import all the functions you want to extend, so Julia knows your intention of adding a method to the function from PlantSimEngine, and not defining your own function. To do so, you have to prefix the said functions by the package name, or import them before *e.g.*: `import PlantSimEngine: inputs_, outputs_` - -So let's do it! Here is our own implementation of the light interception for a `ModelList` component models: - -```@example usepkg -function run!(::Beer, models, status, meteo, constants, extras) - status.PPFD = - meteo.Ri_PAR_f * - exp(-models.light_interception.k * status.LAI) * - constants.J_to_umol -end -``` - -The first argument (`::Beer`) means this method will only execute when the function is called with a first argument that is of type `Beer`. This is our way of telling Julia that this method implements the `Beer` model for the light interception process. - -An important thing to note is that the model parameters are available from the `ModelList` that is passed via the `models` argument. Then parameters are found in field called by the process name, and the parameter name. For example, the `k` parameter of the `Beer` model is found in `models.light_interception.k`. - -One last thing to do is to define the inputs and outputs of our model. This is done by adding a method for the [`inputs`](@ref) and [`outputs`](@ref) functions. These functions take the type of the model as argument, and return a `NamedTuple` with the names of the variables as keys, and their default values as values. - -In our case, the `Beer` model has one input and one output: - -- Inputs: `:LAI`, the leaf area index (m² m⁻²) -- Outputs: `:aPPFD`, the photosynthetic photon flux density (μmol m⁻² s⁻¹) - -Here is how we communicate that to PlantSimEngine: - -```@example usepkg -function PlantSimEngine.inputs_(::Beer) - (LAI=-Inf,) -end - -function PlantSimEngine.outputs_(::Beer) - (aPPFD=-Inf,) -end -``` - -Note that both functions end with an "\_". This is because these functions are internal, they will not be called by the users directly. Users will use [`inputs`](@ref) and [`outputs`](@ref) instead, which call `inputs_` and `outputs_`, but stripping out the default values. - -### Dependencies - -If your model explicitly calls another model, you need to tell PlantSimEngine about it. This is called a hard dependency, in opposition to a soft dependency, which is when your model uses a variable from another model, but does not call it explicitly. - -To do so, we can add a method to the `dep` function that tells PlantSimEngine which processes (and models) are needed for the model to run. - -Our example model does not call another model, so we don't need to implement it. But we can look at *e.g.* the implementation for [`Fvcb`](https://github.com/VEZY/PlantBiophysics.jl/blob/d1d5addccbab45688a6c3797e650a640209b8359/src/processes/photosynthesis/FvCB.jl#L83) in `PlantBiophysics.jl` to see how it works: - -```julia -PlantSimEngine.dep(::Fvcb) = (stomatal_conductance=AbstractStomatal_ConductanceModel,) -``` - -Here we say to PlantSimEngine that the `Fvcb` model needs a model of type `AbstractStomatal_ConductanceModel` in the stomatal conductance process. - -You can read more about dependencies in [Model coupling for modelers](@ref) and [Model coupling for users](@ref). - -### The utility functions - -Before running a simulation, you can do a little bit more for your implementation (optional). - -First, you can add a method for type promotion. It wouldn't make any sense for our example because we have only one parameter. But we can make another example with a new model that would be called `Beer2` that would take two parameters: - -```julia -struct Beer2{T} <: AbstractLight_InterceptionModel - k::T - x::T -end -``` - -To add type promotion to `Beer2` we would do: - -```julia -function Beer2(k,x) - Beer2(promote(k,x)) -end -``` - -This would allow users to instantiate the model parameters using different types of inputs. For example they may use this: - -```julia -Beer2(0.6,2) -``` - -You don't see a problem? Well your users won't either. But there's one: `Beer2` is a parametric type, so all fields share the same type `T`. This is the `T` in `Beer2{T}` and then in `k::T` and `x::T`. And this force the user to give all parameters with the same type. - -And in our example above, the user provides `0.6` for `k`, which is a `Float64`, and `2` for `x`, which is an `Int`. ANd if you don't have type promotion, Julia will return an error because both should be either `Float64` or `Int`. That's were the promotion comes in handy, it will convert all your inputs to a common type (when possible). In our example it will convert `2` to `2.0`. - -A second thing also is to help your user with default values for some parameters (if applicable). For example a user will almost never change the value of `k`. So we can provide a default value like so: - -```@example usepkg -Beer() = Beer(0.6) -``` - -Now the user can call `Beer` with zero value, and `k` will default to `0.6`. - -Another useful thing is the ability to instantiate your model type with keyword arguments, *i.e.* naming the arguments. You can do it by adding the following method: - -```@example usepkg -Beer(;k) = Beer(k) -``` - -Did you notice the `;` before the argument? It tells Julia that we want those arguments provided as keywords, so now we can call `Beer` like this: - -```@example usepkg -Beer(k = 0.7) -``` - -This is nice when we have a lot of parameters and some with default values, but again, this is completely optional. - -The last optional thing to implement is a method for the `eltype` function: - -```@example usepkg -Base.eltype(x::Beer{T}) where {T} = T -``` - -This one helps Julia know the type of the elements in the structure, and make it faster. - -### Traits - -`PlantSimEngine` defines traits to get additional information about the models. At the moment, there are two traits implemented that help the package to know if a model can be run in parallel over space (*i.e.* objects) and/or time (*i.e.* time-steps). - -By default, all models are assumed to be **not** parallelizable over objects and time-steps, because it is the safest default. If your model is parallelizable, you should add the trait to the model. - -For example, if we want to add the trait for parallelization over objects to our `Beer` model, we would do: - -```@example usepkg -PlantSimEngine.ObjectDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsObjectIndependent() -``` - -And if we want to add the trait for parallelization over time-steps to our `Beer` model, we would do: - -```@example usepkg -PlantSimEngine.TimeStepDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsTimeStepIndependent() -``` - -!!! note - A model is parallelizable over objects if it does not call another model directly inside its code. Similarly, a model is parallelizable over time-steps if it does not get values from other time-steps directly inside its code. In practice, most of the models are parallelizable one way or another, but it is safer to assume they are not. - -OK that's it! Now we have a full new model implementation for the light interception process! I hope it was clear and you understood everything. If you think some sections could be improved, you can make a PR on this doc, or open an issue. \ No newline at end of file diff --git a/docs/src/extending/implement_a_process.md b/docs/src/extending/implement_a_process.md deleted file mode 100644 index 5cf1ac8a0..000000000 --- a/docs/src/extending/implement_a_process.md +++ /dev/null @@ -1,152 +0,0 @@ -# Implement a new process - -```@setup usepkg -using PlantSimEngine -using PlantMeteo -PlantSimEngine.@process growth -``` - -## Introduction - -`PlantSimEngine.jl` was designed to make the implementation of new processes and models easy and fast. Let's learn about how to implement a new process with a simple example: implementing a growth model. - -## Implement a process - -To implement a new process, we need to define an abstract structure that will help us associate the models to this process. We also need to generate some boilerplate code, such as a method for the `process` function. Fortunately, PlantSimEngine provides a macro to generate all that at once: [`@process`](@ref). This macro takes only one argument: the name of the process. - -For example, the photosynthesis process in [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) is declared using just this tiny line of code: - -```julia -@process "photosynthesis" -``` - -If we want to simulate the growth of a plant, we could add a new process called `growth`: - -```julia -@process "growth" -``` - -And that's it! Note that the function guides you in the steps you can make after creating a process. Let's break it up here. - -!!! tip - If you know what you're doing, you can directly define a process by hand just by defining an abstract type that is a subtype of `AbstractModel`: - ```julia - abstract type AbstractGrowthModel <: PlantSimEngine.AbstractModel end - ``` - And by adding a method for the `process_` function that returns the name of the process: - ```julia - PlantSimEngine.process_(::Type{AbstractGrowthModel}) = :growth - ``` - But this way, you don't get the nice tutorial adapted to your process 🙃. - -So what you just did is to create a new process called `growth`. By doing so, you created a new abstract structure called `AbstractGrowthModel`, which is used as a supertype of the models. This abstract type is always named using the process name in title case (using `titlecase()`), prefixed with `Abstract` and suffixed with `Model`. - -!!! note - If you don't understand what a supertype is, no worries, you'll understand by seeing the examples below - -## Implement a new model for the process - -To better understand how models are implemented, you can read the detailed instructions from the [next section](@ref model_implementation_page). But for the sake of completeness, we'll implement a growth model here. - -This growth model needs the absorbed photosynthetically active radiation (aPPFD) as an input, and outputs the assimilation, the maintenance respiration, the growth respiration, the biomass increment and the biomass. The assimilation is computed as the product of the aPPFD and the light use efficiency (LUE). The maintenance respiration is a fraction of the assimilation, and the growth respiration is a fraction of the net primary productivity (NPP), which is the assimilation minus the maintenance respiration. The biomass increment is the NPP minus the growth respiration, and the biomass is the sum of the biomass increment and the previous biomass. Note that the previous biomass is always available in the `status` as long as you don't modify it. - -The model is available in the example script [ToyAssimGrowthModel.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyAssimGrowthModel.jl), and is reproduced below: - -```@example usepkg -# Make the struct to hold the parameters, with its documentation: -""" - ToyAssimGrowthModel(Rm_factor, Rg_cost) - ToyAssimGrowthModel(; LUE=0.2, Rm_factor = 0.5, Rg_cost = 1.2) - -Computes the biomass growth of a plant. - -# Arguments - -- `LUE=0.2`: the light use efficiency, in gC mol[PAR]⁻¹ -- `Rm_factor=0.5`: the fraction of assimilation that goes into maintenance respiration -- `Rg_cost=1.2`: the cost of growth maintenance, in gram of carbon biomass per gram of assimilate - -# Inputs - -- `aPPFD`: the absorbed photosynthetic photon flux density, in mol[PAR] m⁻² time-step⁻¹ - -# Outputs - -- `carbon_assimilation`: the assimilation, in gC m⁻² time-step⁻¹ -- `Rm`: the maintenance respiration, in gC m⁻² time-step⁻¹ -- `Rg`: the growth respiration, in gC m⁻² time-step⁻¹ -- `biomass_increment`: the daily biomass increment, in gC m⁻² time-step⁻¹ -- `biomass`: the plant biomass, in gC m⁻² time-step⁻¹ -""" -struct ToyAssimGrowthModel{T} <: AbstractGrowthModel - LUE::T - Rm_factor::T - Rg_cost::T -end - -# Note that ToyAssimGrowthModel is a subtype of AbstractGrowthModel, this is important - -# Instantiate the `struct` with keyword arguments and default values: -function ToyAssimGrowthModel(; LUE=0.2, Rm_factor=0.5, Rg_cost=1.2) - ToyAssimGrowthModel(promote(LUE, Rm_factor, Rg_cost)...) -end - -# Define inputs: -function PlantSimEngine.inputs_(::ToyAssimGrowthModel) - (aPPFD=-Inf,) -end - -# Define outputs: -function PlantSimEngine.outputs_(::ToyAssimGrowthModel) - (carbon_assimilation=-Inf, Rm=-Inf, Rg=-Inf, biomass_increment=-Inf, biomass=0.0) -end - -# Tells Julia what is the type of elements: -Base.eltype(x::ToyAssimGrowthModel{T}) where {T} = T - -# Implement the growth model: -function PlantSimEngine.run!(::ToyAssimGrowthModel, models, status, meteo, constants, extra) - - # The assimilation is simply the absorbed photosynthetic photon flux density (aPPFD) times the light use efficiency (LUE): - status.carbon_assimilation = status.aPPFD * models.growth.LUE - # The maintenance respiration is simply a factor of the assimilation: - status.Rm = status.carbon_assimilation * models.growth.Rm_factor - # Note that we use models.growth.Rm_factor to access the parameter of the model - - # Net primary productivity of the plant (NPP) is the assimilation minus the maintenance respiration: - NPP = status.carbon_assimilation - status.Rm - - # The NPP is used with a cost (growth respiration Rg): - status.Rg = 1 - (NPP / models.growth.Rg_cost) - - # The biomass increment is the NPP minus the growth respiration: - status.biomass_increment = NPP - status.Rg - - # The biomass is the biomass from the previous time-step plus the biomass increment: - status.biomass += status.biomass_increment -end - -# And optionally, we can tell PlantSimEngine that we can safely parallelize our model over space (objects): -PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyAssimGrowthModel}) = PlantSimEngine.IsObjectIndependent() -``` - -Now we can make a simulation as usual: - -```@example usepkg -model = ModelList(ToyAssimGrowthModel(), status = (aPPFD = 20.0,)) -run!(model) -model[:biomass] # biomass in gC m⁻² -``` - -We can also run the simulation over more time-steps: - -```@example usepkg -model = ModelList( - ToyAssimGrowthModel(), - status=(aPPFD=[10.0, 30.0, 25.0],), -) - -run!(model) - -model.status[:biomass] # biomass in gC m⁻² -``` \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md index 855b89038..697124245 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -17,7 +17,7 @@ model = ModelList( status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model ) -run!(model) +out = run!(model) # Define the list of models for coupling: model2 = ModelList( @@ -25,7 +25,7 @@ model2 = ModelList( Beer(0.6), status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model ) -run!(model2, meteo_day) +out2 = run!(model2, meteo_day) ``` @@ -45,21 +45,49 @@ Depth = 5 ## Overview -`PlantSimEngine` is a comprehensive package for simulating and modelling plants, soil and atmosphere. It provides tools to **prototype, evaluate, test, and deploy** plant/crop models at any scale. At its core, PlantSimEngine is designed with a strong emphasis on performance and efficiency. +`PlantSimEngine` is a comprehensive framework for building models of the soil-plant-atmosphere continuum. It includes everything you need to **prototype, evaluate, test, and deploy** plant/crop models at any scale, with a strong emphasis on performance and efficiency, so you can focus on building and refining your models. -The package defines a framework for declaring processes and implementing associated models for their simulation. +**Why choose PlantSimEngine?** -It focuses on key aspects of simulation and modeling such as: +- **Simplicity**: Write less code, focus on your model's logic, and let the framework handle the rest. +- **Modularity**: Each model component can be developed, tested, and improved independently. Assemble complex simulations by reusing pre-built, high-quality modules. +- **Standardisation**: Clear, enforceable guidelines ensure that all models adhere to best practices. This built-in consistency means that once you implement a model, it works seamlessly with others in the ecosystem. +- **Optimised Performance**: Don't re-invent the wheel. Delegating low-level tasks to PlantSimEngine guarantees that your model will benefit from every improvement in the framework. Enjoy faster prototyping, robust simulations, and efficient execution using Julia's high-performance capabilities. -- Easy definition of new processes, such as light interception, photosynthesis, growth, soil water transfer... -- Fast, interactive prototyping of models, with constraints to help users avoid errors, but sensible defaults to avoid over-complicating the model writing process -- No hassle, the package manages automatically input and output variables, time-steps, objects, soft and hard coupling of models with a dependency graph -- Switch between models without changing any code, with a simple syntax to define the model to use for a given process -- Reduce the degrees of freedom by fixing variables, passing measurements, or using a simpler model for a given process -- 🚀(very) fast computation 🚀, think of 100th of nanoseconds for one model, two coupled models (see this [benchmark script](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/benchmark.jl)), or the full energy balance of a leaf using [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) that uses PlantSimEngine -- Out of the box Sequential, Parallel (Multi-threaded) or Distributed (Multi-Process) computations over objects, time-steps and independent processes (thanks to [Floops.jl](https://juliafolds.github.io/FLoops.jl/stable/)) -- Easily scalable, with methods for computing over objects, time-steps and even [Multi-Scale Tree Graphs](https://github.com/VEZY/MultiScaleTreeGraph.jl) -- Composable, allowing the use of any types as inputs such as [Unitful](https://github.com/PainterQubits/Unitful.jl) to propagate units, or [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) to propagate measurement error +## Unique Features + +### Automatic Model Coupling + +**Seamless Integration:** PlantSimEngine leverages Julia's multiple-dispatch capabilities to automatically compute the dependency graph between models. This allows researchers to effortlessly couple models without writing complex connection code or manually managing dependencies. + +**Intuitive Multi-Scale Support:** The framework naturally handles models operating at different scales—from organelle to ecosystem—connecting them with minimal effort and maintaining consistency across scales. + +### Flexibility with Precision Control + +**Effortless Model Switching:** Researchers can switch between different component models using a simple syntax without rewriting the underlying model code. This enables rapid comparison between different hypotheses and model versions, accelerating the scientific discovery process. + +## Batteries included + +- **Automated Management**: Seamlessly handle inputs, outputs, time-steps, objects, and dependency resolution. +- **Iterative Development**: Fast and interactive prototyping of models with built-in constraints to avoid errors and sensible defaults to streamline the model writing process. +- **Control Your Degrees of Freedom**: Fix variables to constant values or force to observations, use simpler models for specific processes to reduce complexity. +- **High-Speed Computations**: Achieve impressive performance with benchmarks showing operations in the 100th of nanoseconds range for complex models (see this [benchmark script](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/benchmark.jl)). +- **Parallelize and Distribute Computing**: Out-of-the-box support for sequential, multi-threaded, or distributed computations over objects, time-steps, and independent processes, thanks to [Floops.jl](https://juliafolds.github.io/FLoops.jl/stable/). +- **Scale Effortlessly**: Methods for computing over objects, time-steps, and [Multi-Scale Tree Graphs](https://github.com/VEZY/MultiScaleTreeGraph.jl). +- **Compose Freely**: Use any types as inputs, including [Unitful](https://github.com/PainterQubits/Unitful.jl) for unit propagation and [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) for measurement error propagation. + +## Performance + +PlantSimEngine delivers impressive performance for plant modeling tasks. On an M1 MacBook Pro: + +- A toy model for leaf area over a year at daily time-scale took only 260 μs (about 688 ns per day) +- The same model coupled to a light interception model took 275 μs (756 ns per day) + +These benchmarks demonstrate performance on par with compiled languages like Fortran or C, far outpacing typical interpreted language implementations. For example, PlantBiophysics.jl, which implements ecophysiological models using PlantSimEngine, has been measured to run up to 38,000 times faster than equivalent implementations in other scientific computing languages. + +## Ask Questions + +If you have any questions or feedback, [open an issue](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues) or ask on [discourse](https://fspm.discourse.group/c/software/virtual-plant-lab). ## Installation @@ -79,7 +107,7 @@ using PlantSimEngine The package is designed to be easy to use, and to help users avoid errors when implementing, coupling and simulating models. -### Simple example +### Simple example Here's a simple example of a model that simulates the growth of a plant, using a simple exponential growth model: @@ -96,9 +124,7 @@ model = ModelList( status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model ) -run!(model) # run the model - -status(model) # extract the status, i.e. the output of the model +out = run!(model) # run the model and extract its outputs ``` > **Note** @@ -110,7 +136,7 @@ Of course you can plot the outputs quite easily: # ] add CairoMakie using CairoMakie -lines(model[:TT_cu], model[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) +lines(out[:TT_cu], out[:LAI], color=:green, axis=(ylabel="LAI (m² m⁻²)", xlabel="Cumulated growing degree days since sowing (°C)")) ``` ### Model coupling @@ -135,9 +161,7 @@ model2 = ModelList( ) # Run the simulation: -run!(model2, meteo_day) - -status(model2) +out2 = run!(model2, meteo_day) ``` The `ModelList` couples the models by automatically computing the dependency graph of the models. The resulting dependency graph is: @@ -159,22 +183,22 @@ The `ModelList` couples the models by automatically computing the dependency gra ╰────────────────────────────────────────────────────────────────╯ ``` -We can plot the results by indexing the model with the variable name (e.g. `model2[:LAI]`): +We can plot the results by indexing the outputs with the variable name (e.g. `out2[:LAI]`): ```@example readme using CairoMakie fig = Figure(resolution=(800, 600)) ax = Axis(fig[1, 1], ylabel="LAI (m² m⁻²)") -lines!(ax, model2[:TT_cu], model2[:LAI], color=:mediumseagreen) +lines!(ax, out2[:TT_cu], out2[:LAI], color=:mediumseagreen) ax2 = Axis(fig[2, 1], xlabel="Cumulated growing degree days since sowing (°C)", ylabel="aPPFD (mol m⁻² d⁻¹)") -lines!(ax2, model2[:TT_cu], model2[:aPPFD], color=:firebrick1) +lines!(ax2, out2[:TT_cu], out2[:aPPFD], color=:firebrick1) fig ``` -### Multiscale modelling +### Multi-scale modeling > See the [Multi-scale modeling](#multi-scale-modeling) section for more details. @@ -186,35 +210,35 @@ mapping = Dict( "Plant" => ( MultiScaleModel( model=ToyLAIModel(), - mapping=[ + mapped_variables=[ :TT_cu => "Scene", ], ), Beer(0.6), MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil"], + mapped_variables=[:soil_water_content => "Soil"], ), MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] ], ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), MultiScaleModel( model=ToyInternodeEmergence(TT_emergence=20.0), - mapping=[:TT_cu => "Scene"], + mapped_variables=[:TT_cu => "Scene"], ), ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), Status(carbon_biomass=1.0) @@ -222,7 +246,7 @@ mapping = Dict( "Leaf" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), Status(carbon_biomass=1.0) @@ -263,7 +287,7 @@ out_vars = Dict( "Soil" => (:soil_water_content,), ) -out = run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()); +out = run!(mtg, mapping, meteo, tracked_outputs=out_vars, executor=SequentialEx()); nothing # hide ``` @@ -271,7 +295,7 @@ We can then extract the outputs in a `DataFrame` and sort them: ```@example readme using DataFrames -df_out = outputs(out, DataFrame) +df_out = convert_outputs(out, DataFrame) sort!(df_out, [:timestep, :node]) ``` @@ -279,6 +303,29 @@ An example output of a multiscale simulation is shown in the documentation of Pl ![Plant growth simulation](www/image.png) +## State of the field + +PlantSimEngine is a state-of-the-art plant simulation software that offers significant advantages over existing tools such as OpenAlea, STICS, APSIM, or DSSAT. + +The use of Julia programming language in PlantSimEngine allows for: + +- Quick and easy prototyping compared to compiled languages +- Significantly better performance than typical interpreted languages +- No need for translation into another compiled language + +Julia's features enable PlantSimEngine to provide: + +- Multiple-dispatch for automatic computation of model dependency graphs +- Type stability for optimized performance +- Seamless compatibility with powerful tools like MultiScaleTreeGraph.jl for multi-scale computations + +PlantSimEngine's approach streamlines the process of model development by automatically managing: + +- Model coupling with automated dependency graph computation +- Time-steps and parallelization +- Input and output variables +- Various types of objects used for simulations (vectors, dictionaries, multi-scale tree graphs) + ## Projects that use PlantSimEngine Take a look at these projects that use PlantSimEngine: @@ -286,8 +333,8 @@ Take a look at these projects that use PlantSimEngine: - [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) - [XPalm](https://github.com/PalmStudio/XPalm.jl) -## Make it yours +## Make it yours -The package is developed so anyone can easily implement plant/crop models, use it freely and as you want thanks to its MIT license. +The package is developed so anyone can easily implement plant/crop models, use it freely and as you want thanks to its MIT license. -If you develop such tools and it is not on the list yet, please make a PR or contact me so we can add it! 😃 \ No newline at end of file +If you develop such tools and it is not on the list yet, please make a PR or contact me so we can add it! 😃 diff --git a/docs/src/introduction/why_julia.md b/docs/src/introduction/why_julia.md new file mode 100644 index 000000000..892421fc7 --- /dev/null +++ b/docs/src/introduction/why_julia.md @@ -0,0 +1,94 @@ +# The choice of using Julia + +PlantSimEngine is implemented in Julia. It arose from a particular combination of [needs and requirements](why_plantsimengine.md) that Julia addresses effectively. + +Other modelling frameworks, FSPMs and crop models are -often- written in combinations of Java, C++, Python, or Fortran. Given that Julia isn't the language many researchers (and developers!) are most familiar with yet, this page provides a short explanation of the reasoning behind that language choice. Another nice resource is [this discourse post](https://fspm.discourse.group/t/why-is-julia-meant-for-fspm/175) by Alejandro Morales Sierra, the creator and maintainer of Virtual Plant Lab. + +## From research to real-world applications + +PlantSimEngine was originally a goal-oriented framework. Its features arose -and continue to evolve- out of necessity for more and more complex simulation setups. + +While PlantSimEngine primarily helps researchers prototype and test their models efficiently, we consistently work with the vision of making it suitable for real-world applications. Our goal is to build a bridge between academic plant modeling and practical field applications. Ideally, researchers should be able to develop and refine their models in a comfortable environment, and these models could eventually be deployed in production environments. + +This vision of dual-purpose functionality drives our focus on performance optimization. We aspire for the models you develop to be useful beyond academic papers, potentially serving reliably in production environments where efficiency and accuracy are crucial. Julia's strong performance characteristics support this vision in ways other languages would struggle to match. + +PlantSimEngine aims to balance scientific rigor with developer productivity, with the long-term goal of ensuring that models can be deployed at scale. Julia provides an environment where researchers can express complex mathematical concepts directly in code with good performance potential, creating a pathway for these models to potentially reach practical implementation. + +## PlantSimEngine's constraints + +### Performance + +While computers have gained several orders of magnitude of power and memory over the past few decades, to the point where many prior performance bottlenecks have vanished, performance can still be a limiting factor. + +Simulating multiple processes with user-provided variables over many plants with tens of thousands of leaves requires a lot of computation. Using a higher-level language such as Python or R would not lead to adequate simulation times. + +In fact, part of the initial motivation to commit to Julia happened after porting [a model](https://github.com/VEZY/DynACof.jl) from R to Julia and getting several orders of magnitude difference in performance 'out-of-the-box'. Seeing computations that previously took minutes suddenly completing in seconds was quite convincing (see also [this benchmark](https://vezy.github.io/PlantBiophysics-paper/notebooks_performance_Fig5_PlantBiophysics_performance/) showing a difference of 5 orders of magnitude). + +Julia, with its well-designed 'Just-ahead-of-time' compilation model and its flexibility allowing to do some lower-level optimisation, doesn't suffer from the limitations one would encounter by using only Python or R. + +### Flexibility, ease of use + +PlantSimEngine was also developed with a few goals in mind, one of them being to make hypothesis testing quite easy. It is currently difficult to validate FSPM, crop model or ecophysiological hypotheses in many existing frameworks due to their rigid structure or steep learning curve. + +Similarly, when developing a full-featured FSPM, there might be a need to test different models for a specific process, or to switch a model for a more complex one. API and language ease of use is as much of a factor as automated model coupling in keeping these changes smooth. + +### Packages destined to be used by a wider community + +As mentioned earlier, PlantSimEngine is intended for a wide audience. Only few of them are expected to have a strong development background. Many other potential users might be researchers more well-versed in ecophysiology or plant architecture and only know a little bit of Python, Matlab or R. Reducing friction for these users is paramount. + +Open-source libraries/packages, ease of installation and low entry barrier also factor in the decision. + +### Modularity and flexibility while retaining performance + +One approach could be to combine, say, Python, with a more performant language such as C++ or Fortran. The slower but flexible language being used for prototyping, and when performance is required, some chunks are reimplmented in the other language. + +This fits the performance constraint, but has a few caveats. + +### Low developer bandwidth + +And of course, budget, time and resources are a concern. The more autonomous researchers and modelers are, and the less specialist developer/engineering resources are required, the easier it is for the project to keep evolving. + +## Comparison + +### The Two-Language Problem + +Combining two different languages requires a lot of language expertise, with constant knowledge refreshing, as one might only occasionally work with and debug with the lower-level language. Or more engineering resources. + +Speed of iteration is also lost whenever performance is a concern, which happens often in our context. However modular and easy-to-use a language like Python might be, whenever it's time to switch to a low-level language, development speed will slow down. + +Julia effectively solves this problem. While it might be a little harder to learn than Python, and require extra knowledge to properly make use of its flexibility and performance capabilities, it leads to a smoother development experience. + +Everything can be done using Julia exclusively, so there is no need to learn two languages. No need to interface between them. Iteration speed doesn't suddenly grind to a halt if a low-level implementation is needed. A competent researcher-developer can move seamlessly from prototype to production, while still being able to focus on modeling and the actual plant side of things. + +![Language usage comparison for different ML packages (source: https://pde-on-gpu.vaw.ethz.ch/lecture1/)](../www/l1_flux-vs-tensorflow.png) +(Language usage comparison for different ML packages; source: https://pde-on-gpu.vaw.ethz.ch/lecture1/) + +It seems we aren't the only ones to find Julia a good tool for our job. Other niches where Julia is gaining traction tend to be other computationally heavy areas with much active research, such as machine learning and climate modeling - areas where this balance of expressivity and performance is equally valuable. + +### A good balance in terms of accessibility + +Another argument in favour of Julia is that one of the aims for PlantSimEngine is to be easy-to-use for researchers wishing to test hypothesis, or reproduce results from other papers. Scientific reproducibility is greatly enhanced when the barrier to running and modifying simulations is lowered. + +Many researchers are not developers by trade or heart, and a Java-only or C++-only implementation, on top of the earlier points, would not be accessible enough and would not gain much traction. + +Julia, while less ubiquitous than other languages in research circles, resembles Python and R and is more beginner-friendly than Java or C++. It is easier for a Python user to learn to use a simple Julia package than a C++ one. + +Users will also find it easier to quickly implement new models without the potential hurdle of a low-level implementation, or some language interfacing also being required. The prototyping phase doesn't require a subsequent performance tuning phase. + +### Ease of environment setup + +Similarly, Julia's language and package installation is -mostly- fairly straightforward and requires little additional knowledge. + +The package manager is built directly into the language, making dependency management straightforward. This is particularly important for reproducible scientific workflows, where consistent environments are crucial. + +### Downsides acceptable + +While very practical for a 'researcher-developer', Julia is of course far from being the perfect language in every discipline. It is massive in terms of features, has a heavy runtime, is more involved to learn and master quickly compared to Python, has a few hurdles for beginners, some quirks that can be awkward for developers, tools that aren't fully mature, no clear 'recommended' workflow, and so on. + +The cost for switching may not be worth it in many other circumstances. However, several of these downsides, while very relevant for embedded systems, or game development, are much less relevant regarding PlantSimEngine. And others can be mitigated with, hopefully, adequate learning resources and documentation. + +## Conclusion + +For PlantSimEngine's specific requirements—balancing performance with flexibility, enabling rapid iteration while maintaining computational efficiency, and providing an accessible interface for both researchers and field practitioners—Julia represents a suitable choice. The language allows us to build an ecosystem where plant modeling can advance through collaborative, efficient, and scientifically rigorous development while delivering real-world value through production deployments. + +While no language solution is perfect, Julia's combination of features makes it well-suited to the challenges of modern plant modeling and simulation, both in research and practical applications. We're optimistic about the possibilities it offers for the future of plant modeling. diff --git a/docs/src/introduction/why_plantsimengine.md b/docs/src/introduction/why_plantsimengine.md new file mode 100644 index 000000000..ed8759658 --- /dev/null +++ b/docs/src/introduction/why_plantsimengine.md @@ -0,0 +1,107 @@ +# Why PlantSimEngine? + +PlantSimEngine was developed to address fundamental limitations in existing plant modeling tools. This framework emerged from the need for a system that could efficiently handle the complex dynamics of the soil-plant-atmosphere continuum while remaining accessible to researchers and practitioners from diverse disciplines. + +## The Current Landscape of Plant Modeling + +Plant modeling has evolved significantly over the years, with different tools making different design tradeoffs to address specific research needs. These tools generally fall into three categories, each with their own strengths and limitations: + +### Monolithic Systems + +Systems like APSIM[^1], GroIMP[^2], AMAPStudio[^3], Helios[^4], and CPlantBox[^5] offer comprehensive functionality but present certain tradeoffs: + +These systems provide robust, well-tested frameworks with established scientific validity, but their large, complex codebases can be challenging to navigate and modify without extensive programming expertise. + +Their comprehensive architecture offers a wealth of integrated features but may require adaptation when implementing novel approaches that don't align with their predefined frameworks. + +They excel at specific types of simulations but may require additional engineering effort for seamless multi-scale simulations and model coupling across the soil-plant-atmosphere continuum. + +These platforms typically require dedicated engineering resources for maintenance and extension, with research teams often needing specialized technical staff to implement new models. + +### Distributed Systems + +Platforms like OpenAlea[^6] and Crops in Silico[^7] offer different advantages and tradeoffs: + +These systems provide accessible interfaces (often in Python) that prioritize ease of use and flexibility, making them approachable for many researchers, though they may require performance optimization for large-scale simulations. + +Their modular nature facilitates component reuse and integration, while sometimes requiring proficiency in multiple programming languages for extending computational backends. + +They support diverse modeling paradigms but may involve a longer iteration cycle between design, implementation, and performance tuning compared to more specialized tools. + +While offering flexibility, implementing complex models often requires significant developer time, especially when optimizing performance using lower-level languages. + +### Architecture-Focused Tools + +Tools like AMAPSim[^8] make specific design choices that benefit certain applications: + +These systems excel in their focused domains (such as structural modeling of plants) while requiring integration with other tools for comprehensive studies of plant physiology and environmental responses. + +Their implementation in languages like C++ or Java delivers excellent performance but represents a tradeoff in terms of accessibility for researchers without expertise in these languages. + +They provide sophisticated functionality in their target domains but may require additional work for rapid hypothesis testing and model prototyping across diverse aspects of plant science. + +## The PlantSimEngine Solution + +PlantSimEngine brings together innovative ideas to address these various tradeoffs, offering a unique combination of features: + +### Automatic Model Coupling + +**Seamless Integration:** PlantSimEngine leverages Julia's multiple-dispatch capabilities to automatically compute the dependency graph between models. This allows researchers to effortlessly couple models without writing complex connection code or manually managing dependencies. + +**Intuitive Multi-Scale Support:** The framework naturally handles models operating at different scales—from organelle to ecosystem—connecting them with minimal effort and maintaining consistency across scales. + +### Flexibility with Precision Control + +**Effortless Model Switching:** Researchers can switch between different component models using a simple syntax without rewriting the underlying model code. This enables rapid comparison between different hypotheses and model versions, accelerating the scientific discovery process. + +**Fine-Grained Model Control:** PlantSimEngine allows users to fix parameters, force variables to match observed values, or select simpler models for specific processes. This flexibility helps reduce overall system complexity while maintaining precision where it matters most. + +**Adaptive Scalability:** The same framework efficiently supports both simple prototypes for single-plant studies and complex ecosystem simulations, scaling computational resources appropriately to the problem at hand. + +### Outstanding Performance + +**High-Speed Computation:** Benchmarks demonstrate operations completing in hundreds of nanoseconds, making PlantSimEngine suitable for computationally intensive applications. For example, the [PlantBiophysics.jl implementation is over 38,000 times faster](https://vezy.github.io/PlantBiophysics-paper/notebooks_performance_Fig5_PlantBiophysics_performance/) than equivalent implementations in R. + +**Computational Efficiency:** Julia's just-ahead-of-time compilation and native support for parallelism ensure that optimizations made during prototyping directly transfer to larger-scale applications, eliminating the need for reimplementation in a different language for performance gains. + +### Developer Efficiency + +**Reduced Implementation Time:** PlantSimEngine leverages Julia's dynamic language features while maintaining the performance of statically-compiled languages. This significantly reduces the time researchers spend implementing and optimizing models. + +**Modular Building Blocks:** The component-based architecture allows models to be built as unit components that can be stacked like building blocks to create complex systems. This modularity dramatically increases code reuse and reduces redundant implementation efforts. + +**No Engineering Overhead:** Unlike monolithic systems that require dedicated engineering teams or distributed platforms that need backend optimization, PlantSimEngine enables domain scientists to independently develop high-performance models without specialized programming expertise. + +**Rapid Prototyping to Production:** The same code used for quick prototyping can transition directly to production-scale simulations without rewriting, eliminating the traditional gap between exploratory research and application. + +## Key Innovations + +PlantSimEngine's approach to plant modeling represents a paradigm shift in how scientists can build and use models: + +- **Uniform API:** Standardized interfaces make it easy to define new processes and component models, reducing the cognitive load on researchers. + +- **Automatic Dependency Resolution:** The system automatically determines the relationships between different models and processes, eliminating the need for manual coupling. + +- **Seamless Parallelization:** Out-of-the-box support for parallel and distributed computation allows researchers to focus on the science rather than implementation details. + +- **Flexible Model Integration:** The ability to easily combine models from different sources and at different scales facilitates more comprehensive and realistic simulations. + +- **User-Centric Design:** Emphasizing usability ensures that researchers with varied programming backgrounds can effectively engage with the system. + +By offering solutions to the various tradeoffs present in existing modeling approaches, PlantSimEngine enables researchers to focus more on scientific questions and less on technical implementation details, accelerating the pace of discovery in plant science, agronomy, and related fields. + +[^1]: Holzworth, D. P. et al. APSIM – Evolution towards a new generation of agricultural systems simulation. Environmental Modelling & Software 62, 327-350 (2014). + +[^2]: Hemmerling, R., Kniemeyer, O., Lanwert, D., Kurth, W. & Buck-Sorlin, G. The rule-based language XL and the modelling environment GroIMP illustrated with simulated tree competition. Funct. Plant Biol. 35, 739 (2008). + +[^3]: Griffon, S., and de Coligny, F. AMAPstudio: An editing and simulation software suite for plants architecture modelling. Ecological Modelling 290 (2014): 3‑10. . + +[^4]: Bailey, R. Spatial Modeling Environment for Enhancing Conifer Crown Management. Front. For. Glob. Change 3, 106 (2020). + +[^5]: Schnepf, A., Leitner, D., Landl, M., Lobet, G., Mai, T. H., Morandage, S., Sheng, C., Zörner, M., Vanderborght, J., & Vereecken, H. CPlantBox: A whole-plant modelling framework for the simulation of water- and carbon-related processes. in silico Plants, 63 (2018). + +[^6]: Pradal, C. et al. OpenAlea: A visual programming and component-based software platform for plant modeling. Funct. Plant Biol. 35, 751-760 (2008). + +[^7]: Marshall-Colon, A. et al. Crops In Silico: Generating Virtual Crops Using an Integrative and Multi-Scale Modeling Platform. Frontiers in Plant Science 8 (2017). . + +[^8]: Barczi, J.-F., Rey, H., Caraglio, Y., Reffye, P. de, Barthélémy, D., Dong, Q. X., & Fourcaud, T. AmapSim: A Structural Whole-plant Simulator Based on Botanical Knowledge and Designed to Host External Functional Models. Annals of botany, 101(8), 1125-1138 (2008). diff --git a/docs/src/model_coupling/model_coupling_modeler.md b/docs/src/model_coupling/model_coupling_modeler.md deleted file mode 100644 index 1ed852df8..000000000 --- a/docs/src/model_coupling/model_coupling_modeler.md +++ /dev/null @@ -1,173 +0,0 @@ -# Model coupling for modelers - -```@setup usepkg -using PlantSimEngine, PlantMeteo -# Import the example models defined in the `Examples` sub-module: -using PlantSimEngine.Examples - -m = ModelList( - Process1Model(2.0), - Process2Model(), - Process3Model(), - Process4Model(), - Process5Model(), - Process6Model(), - Process7Model(), -) -``` - -This section uses notions from the previous section. If you are not familiar with the concepts of model coupling in PlantSimEngine, please read the previous section first: [Model coupling for users](@ref). - -## Hard coupling - -A model that calls explicitly another process is called a hard-coupled model. It is implemented by calling the process function directly. - -Let's go through the example processes and models from a script provided by the package here [examples/dummy.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/dummy.jl) - -In this script, we declare seven processes and seven models, one for each process. The processes are simply called "process1", "process2"..., and the model implementations are called `Process1Model`, `Process2Model`... - -`Process2Model` calls `Process1Model` explicitly, which defines `Process1Model` as a hard-dependency of `Process2Model`. The is as follows: - -```julia -function PlantSimEngine.run!(::Process2Model, models, status, meteo, constants, extra) - # computing var3 using process1: - run!(models.process1, models, status, meteo, constants) - # computing var4 and var5: - status.var4 = status.var3 * 2.0 - status.var5 = status.var4 + 1.0 * meteo.T + 2.0 * meteo.Wind + 3.0 * meteo.Rh -end -``` - -We see that coupling a model (`Process2Model`) to another process (`process1`) is done by calling the `run!` function again. The `run!` function is called with the same arguments as the `run!` function of the model that calls it, except that we pass the process we want to simulate as the first argument. - -!!! note - We don't enforce any type of model to simulate `process1`. This is the reason why we can switch so easily between model implementations for any process, by just changing the model in the `ModelList`. - -A hard-dependency must always be declared to PlantSimEngine. This is done by adding a method to the `dep` function. For example, the hard-dependency to `process1` into `Process2Model` is declared as follows: - -```julia -PlantSimEngine.dep(::Process2Model) = (process1=AbstractProcess1Model,) -``` - -This way PlantSimEngine knows that `Process2Model` needs a model for the simulation of the `process1` process. Note that we don't add any constraint to the type of model we have to use (we use `AbstractProcess1Model`), because we want any model implementation to work with the coupling, as we only are interested in the value of a variable, not the way it is computed. - -Even if it is discouraged, you may have a valid reason to force the coupling with a particular model, or a kind of models though. For example, if we want to use only `Process1Model` for the simulation of `process1`, we would declare the dependency as follows: - -```julia -PlantSimEngine.dep(::Process2Model) = (process1=Process1Model,) -``` - -## Soft coupling - -A model that takes outputs of another model as inputs is called a soft-coupled model. There is nothing to do on the modeler side to declare a soft-dependency. The detection is done automatically by PlantSimEngine using the inputs and outputs of the models. - -## Handling dependencies in a multiscale context - - If a model requires some input variable that is computed at another scale, providing the appropriate mapping will resolve name conflicts and enable proper use of that variable and there will be no extra steps for the user or the modeler. - - In the case of a hard dependency that operates at a different scale from its parent, the same principle applies and there are also no extra steps on the user-side. - - On the other hand, modelers need to bear in mind a couple of subtleties when developing models that possess hard dependencies that operate at a different organ level from their parent : - - The parent model directly handles the call to its hard dependency model(s), meaning they are not explicitely managed by the dependency graph. - Therefore only the owning model of that dependency is visible in the graph, and its hard dependency nodes are internal. - - When the caller (or any downstream model that requires some variables from the hard dependency) operates at the same scale, variables are easily accessible, and no mapping is required. - - If an inner model operates at a different scale/organ level, a modeler must declare hard dependencies with their respective organ level, similarly to the way the user provides a mapping. - - Conceptually : - -```julia - PlantSimEngine.dep(m::ParentModel) = ( - name_provided_in_the_mapping=AbstractHardDependencyModel => ["Organ_Name_1",], -) -``` - - Here's a concrete example in [XPalm](https://github.com/PalmStudio/XPalm.jl), an oil palm model developed on top of PlantSimEngine. - Organs are produced at the phytomer scale, but need to run an age model and a biomass model at the reproductive organs' scales. - -```julia - PlantSimEngine.dep(m::ReproductiveOrganEmission) = ( - initiation_age=AbstractInitiation_AgeModel => [m.male_symbol, m.female_symbol], - final_potential_biomass=AbstractFinal_Potential_BiomassModel => [m.male_symbol, m.female_symbol], -) -``` - -The user-mapping includes the required models at specific organ levels. Here's the relevant portion of the mapping for the male reproductive organ : - -```julia -mapping = Dict( - ... - "Male" => - MultiScaleModel( - model=XPalm.InitiationAgeFromPlantAge(), - mapping=[:plant_age => "Plant",], - ), - ... - XPalm.MaleFinalPotentialBiomass( - p.parameters[:male][:male_max_biomass], - p.parameters[:male][:age_mature_male], - p.parameters[:male][:fraction_biomass_first_male], - ), - ... -) -``` - -The model's constructor provides convenient default names for the scale corresponding to the reproductive organs. A user may override that if their naming schemes or MTG attributes differ. - -```julia -function ReproductiveOrganEmission(mtg::MultiScaleTreeGraph.Node; phytomer_symbol="Phytomer", male_symbol="Male", female_symbol="Female") - ... -end -``` - -But how does a model M calling a hard dependency H provide H's variables when calling H's `run!` function ? The status the user provides M operates at M's organ level, so if used to call H's run! function any required variable for H will be missing. - -PlantSimEngine provides what are called Status Templates in the simulation graph. Each organ level has its own Status template listing the available variables at that scale. -So when a model M calls a hard dependency H's `run!` function, any required variables can be accessed through the status template of H's organ level. - -Using the same example in XPalm : - -```julia -# Note that the function's 'status' parameter does NOT contain the variables required by the hard dependencies as the calling model's organ level is "Phytomer", not "Male" or "Female" - -function PlantSimEngine.run!(m::ReproductiveOrganEmission, models, status, meteo, constants, sim_object) - ... - status.graph_node_count += 1 - - # Create the new organ as a child of the phytomer: - st_repro_organ = add_organ!( - status.node[1], # The phytomer's internode is its first child - sim_object, # The simulation object, so we can add the new status - "+", status.sex, 4; - index=status.phytomer_count, - id=status.graph_node_count, - attributes=Dict{Symbol,Any}() - ) - - # Compute the initiation age of the organ: - PlantSimEngine.run!(sim_object.models[status.sex].initiation_age, sim_object.models[status.sex], st_repro_organ, meteo, constants, sim_object) - PlantSimEngine.run!(sim_object.models[status.sex].final_potential_biomass, sim_object.models[status.sex], st_repro_organ, meteo, constants, sim_object) -end -``` - -In the above example the organ and its status template are created on the fly. -When that isn't the case, the status template can be accessed through the simulation graph : - -```julia -function PlantSimEngine.run!(m::ReproductiveOrganEmission, models, status, meteo, constants, sim_object) - - ... - - if status.sex == "Male" - - status_male = sim_object.statuses["Male"][1] - run!(sim_object.models["Male"].initiation_age, models, status_male, meteo, constants, sim_object) - run!(sim_object.models["Male"].final_potential_biomass, models, status_male, meteo, constants, sim_object) - else - # Female - ... - end -end -``` \ No newline at end of file diff --git a/docs/src/model_coupling/multiscale.md b/docs/src/model_coupling/multiscale.md deleted file mode 100644 index 750fbad7d..000000000 --- a/docs/src/model_coupling/multiscale.md +++ /dev/null @@ -1,372 +0,0 @@ -# Multi-scale modeling - -## What is multi-scale modeling? - -Multi-scale modeling is the process of simulating a system at multiple levels of detail simultaneously. For example, some models can run at the organ scale while others run at the plot scale. Each model can access variables at its scale and other scales if needed, allowing for a more comprehensive system representation. It can also help identify emergent properties that are not apparent at a single level of detail. - -For example, a model of photosynthesis at the leaf scale can be combined with a model of carbon allocation at the plant scale to simulate the growth and development of the plant. Another example is a combination of models to simulate the energy balance of a forest. To simulate it, you need a model for each organ type of the plant, another for the soil, and finally, one at the plot scale, integrating all others. - -PlantSimEngine provides a framework for multi-scale modeling to seamlessly integrate models at different scales, keeping all nice functionalities provided at one scale. A nice feature is that models do not need to be aware of the scale at which they are running, nor about the scales at which their inputs are computed, or outputs will be given, which means the model can be reused at different scales or no scale. - -PlantSimEngine automatically computes the dependency graph between mono and multi-scale models, considering every combination of models at any scale, to determine the order of model execution. This means that the user does not need to worry about the order of model execution and can focus on the model definition and the mapping between models and scales. - -Using PlantSimEngine for multi-scale modeling is relatively easy and follows the same rules as mono-scale models. Let's dive into the details with a short tutorial. - -## Simple mapping between models and scales - -To get started, we have to define a mapping between models and scales. - -Let's import the `PlantSimEngine` package and example models we will use in this tutorial: - -```@example usepkg -using PlantSimEngine -using PlantSimEngine.Examples # Import some example models -``` - -!!! note - The `Examples` submodule exports a few simple models we will use in this tutorial. The models are also found in the `examples` folder of the package. - -We now have access to models for the simulation of different processes. We can associate each model with a scale by defining a mapping between models and scales. The mapping is a dictionary with the name of the scale as the key and the model as the value. For example, we can define a mapping to simulate the assimilation process at the leaf scale with `ToyAssimModel` as follows: - -```@example usepkg -mapping = Dict("Leaf" => ToyAssimModel()) -``` - -In this example, the dictionary's key is the name of the scale (`"Leaf"`), and the value is the model. The model is an example model provided by `PlantSimEngine`, so we must prefix it with the module name. - -We can check if the mapping is valid by calling `to_initialize`: - -```@example usepkg -to_initialize(mapping) -``` - -The `to_initialize` function checks if models from any scale need further initialization before simulation. This is the case when some input variables of the model are not computed by another model. In this example, the `ToyAssimModel` needs `:aPPFD` and `:soil_water_content` as inputs. To run a simulation, we must provide a value for the variables or a model that simulates them. - -The initialization values for the variables can be provided using the `Status` type along with the model, *e.g.*: - -```@example usepkg -mapping = Dict( - "Leaf" => ( - ToyAssimModel(), - Status(aPPFD=1300.0, soil_water_content=0.5), - ), -) -``` - -!!! note - The model and the `Status` are provided as a `Tuple` to the `"Leaf"` scale. - -If we re-execute `to_initialize`, we get an empty dictionary, meaning the mapping is valid, and we can start the simulation: - -```@example usepkg -to_initialize(mapping) -``` - -## Multiscale mapping between models and scales - -In our previous example, we provided the value for the `soil_water_content` variable. However, we could also provide a model that simulates it at the soil scale. The only difference now is that we have to tell PlantSimEngine that our -`ToyAssimModel` is now multiscale and takes the `soil_water_content` variable from the `"Soil"` scale. We can do that by wrapping the `ToyAssimModel` in a `MultiScaleModel`: - -```@example usepkg -mapping = Dict( - "Soil" => ToySoilWaterModel(), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil" => :soil_water_content,], - ), - Status(aPPFD=1300.0), - ), -); -nothing # hide -``` - -The `MultiScaleModel` takes two arguments: the model and the mapping between the model and the scales. The mapping is a vector of pairs of pairs mapping the variable's name with the name of the scale its value comes from, and the name of the variable at that scale. In this example, we map the `soil_water_content` variable at scale "Leaf" to the `soil_water_content` variable at the `"Soil"` scale. If the name of the variable is the same between both scales, we can omit the variable name at the origin scale, *e.g.* `[:soil_water_content => "Soil"]`. - -!!! note - The variable `aPPFD` is still provided in the `Status` type as a constant value. - -We can check again if the mapping is valid by calling `to_initialize`: - -```@example usepkg -to_initialize(mapping) -``` - -`to_initialize` returns an empty dictionary, meaning the mapping is valid. - -## More on MultiScaleModel - -`MultiScaleModel` is a wrapper around a model that allows it to take inputs or give outputs from other scales. It takes two arguments: the model and the mapping between the model and the scales. The mapping is a vector of pairs of pairs mapping the variable's name with the name of the scale its value comes from, and its name at that scale. - -The variable can map a single value if there is only one node to map to or a vector of values if there are several. It can also map to several types of nodes at the same time. - -Let's take a look at a more complex example of a mapping: - -```@example usepkg -mapping = Dict( - "Scene" => ToyDegreeDaysCumulModel(), - "Plant" => ( - MultiScaleModel( - model=ToyLAIModel(), - mapping=[ - :TT_cu => "Scene", - ], - ), - Beer(0.6), - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - :carbon_assimilation => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - MultiScaleModel( - model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], - ), - ), - "Internode" => ( - MultiScaleModel( - model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], - ), - ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - Status(carbon_biomass=1.0), - ), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil", :aPPFD => "Plant"], - ), - MultiScaleModel( - model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], - ), - ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - Status(carbon_biomass=0.5), - ), - "Soil" => ( - ToySoilWaterModel(), - ), -); -nothing # hide -``` - -In this example, we expect to make a simulation at five different scales: `"Scene"`, `"Plant"`, `"Internode"`, `"Leaf"`, and `"Soil"`. The `"Scene"` scale represents the whole scene, where one or several plants can live. The `"Plant"` scale is, well, the whole plant scale, `"Internode"` and `"Leaf"` are organ scales, and `"Soil"` is the soil scale. This mapping is used to compute the carbon allocation (`ToyCAllocationModel`) to the different organs of the plant (`"Leaf"` and `"Internode"`) from the assimilation at the `"Leaf"` scale (*i.e.* the offer) and their carbon demand (`ToyCDemandModel`). The `"Soil"` scale is used to compute the soil water content (`ToySoilWaterModel`), which is needed to calculate the assimilation at the `"Leaf"` scale (`ToyAssimModel`). We also can note that we compute the maintenance respiration at the `"Leaf"` and `"Internode"` scales (`ToyMaintenanceRespirationModel`), which is summed up to compute the total maintenance respiration at the `"Plant"` scale (`ToyPlantRmModel`). - -We see that all scales are interconnected, with computations at the organ scale that may depend on the soil scale and at the plant scale that depends on the organ scale and scene scale. - -Something important to note here is that we have different ways to define the mapping for the `MultiScaleModel`. For example, we have `:carbon_assimilation => ["Leaf"]` at the plant scale for `ToyCAllocationModel`. This mapping means that the variable `carbon_assimilation` is mapped to the `"Leaf"` scale. However, we could also have `:carbon_assimilation => "Leaf"`, which is not completely equivalent. - -!!! note - Note the difference between `:carbon_assimilation => ["Leaf"]` and `:carbon_assimilation => "Leaf"` is that "Leaf" is given as a vector in the first definition, and as a scalar in the second one. - -The difference is that the first one maps to a vector of values, while the second one maps to a single value. The first one is useful when we don't know how many nodes there will be in the plant of type `"Leaf"`. In this case, the values are available as a vector in the `carbon_assimilation` variable of the `status` inside the model. The second one should only be used if we are sure that there will be only one node at this scale, and in this case, the one and single value is given as a scalar in the `carbon_assimilation` variable of the `status` inside the model. - -A third form for the mapping would be `:carbon_assimilation => ["Leaf", "Internode"]`. This form is useful when we need values for a variable from several scales simultaneously. In this case, the values are available as a vector in the `carbon_assimilation` variable of the `status` inside the model, sorted in the same order as nodes are traversed in the graph. - -A last form is to map to a specific variable name at the target scale, *e.g.* `:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm]`. This form is useful when the variable name is different between scales, and we want to map to a specific variable name at the target scale. In this example, the variable `Rm_organs` at plant scale takes its values (is mapped) from the variable `Rm` at the `"Leaf"` and `"Internode"` scales. - -## Running a simulation - -Now that we have a valid mapping, we can run a simulation. Running a multiscale simulation requires two more things compared to what we saw previously: a plant graph and the definition of the output variables we want dynamically for each scale. - -### Plant graph - -We can import an example multi-scale tree graph like so: - -```@example usepkg -mtg = import_mtg_example() -``` - -!!! note - You can use `import_mtg_example` only if you previously imported the `Examples` sub-module of PlantSimEngine, *i.e.* `using PlantSimEngine.Examples`. - -This graph has a root node that defines a scene, then a soil, and a plant with two internodes and two leaves. - -### Output variables - -Models can access only one time step at a time, so the output at the end of a simulation is only the last time step. However, we can define a list of variables we want to get dynamically for each time step and each scale. This list is given as a dictionary with the name of the scale as the key and a vector of variables as the value. For example, we can define a list of variables we want to get at each time step for different scales as follows: - -```@example usepkg -outs = Dict( - "Scene" => (:TT, :TT_cu,), - "Plant" => (:aPPFD, :LAI), - "Leaf" => (:carbon_assimilation, :carbon_demand, :carbon_allocation, :TT), - "Internode" => (:carbon_allocation,), - "Soil" => (:soil_water_content,), -) -``` - -These variables will be available in the `outputs` field of the simulation object, with a value for each time step. - -### Meteorological data - -As for mono-scale models, we need to provide meteorological data to run a simulation. We can use the `PlantMeteo` package to generate some dummy data for two time steps: - -```@example usepkg -meteo = Weather( - [ - Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f = 200.0), - Atmosphere(T=25.0, Wind=0.5, Rh=0.8, Ri_PAR_f = 180.0) -] -) -``` - -### Simulation - -Let's make a simulation using the graph and outputs we just defined: - -```@example usepkg -sim = run!(mtg, mapping, meteo, outputs = outs); -nothing # hide -``` - -And that's it! - -We can now access the outputs for each scale as a dictionary of vectors of values per variable and scale like this: - -```@example usepkg -outputs(sim); -nothing # hide -``` - -Or as a `DataFrame` using the `DataFrames` package: - -```@example usepkg -using DataFrames -outputs(sim, DataFrame) -``` - -The values for the last time-step of the simulation are also available from the statuses: - -```@example usepkg -status(sim); -nothing # hide -``` - -This is a dictionary with the scale as the key and a vector of `Status` as values, one per node of that scale. So, in this example, the `"Leaf"` scale has two nodes, so the value is a vector of two `Status` objects, and the `"Soil"` scale has only one node, so the value is a vector of one `Status` object. - - -## Avoiding cyclic dependencies - -When defining a mapping between models and scales, it is important to avoid cyclic dependencies. A cyclic dependency occurs when a model at a given scale depends on a model at another scale that depends on the first model. Cyclic dependencies are bad because they lead to an infinite loop in the simulation (the dependency graph keeps cycling indefinitely). - -PlantSimEngine will detect cyclic dependencies and raise an error if one is found. The error message indicates the models involved in the cycle, and the model that is causing the cycle will be highlighted in red. - -For example the following mapping will raise an error: - -!!! details - Example mapping - - ```julia - mapping_cyclic = Dict( - "Plant" => ( - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - :carbon_demand => ["Leaf", "Internode"], - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - MultiScaleModel( - model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], - ), - Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6), - ), - "Internode" => ( - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - Status(TT=10.0, carbon_biomass=1.0), - ), - "Leaf" => ( - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - ToyCBiomassModel(1.2), - Status(TT=10.0), - ) - ) - ``` - -Let's see what happens when we try to build the dependency graph for this mapping: - -```julia -julia> dep(mapping_cyclic) -ERROR: Cyclic dependency detected in the graph. Cycle: - Plant: ToyPlantRmModel - └ Leaf: ToyMaintenanceRespirationModel - └ Leaf: ToyCBiomassModel - └ Plant: ToyCAllocationModel - └ Plant: ToyPlantRmModel - - You can break the cycle using the `PreviousTimeStep` variable in the mapping. -``` - -How can we interpret the message? We have a list of five models involved in the cycle. The first model is the one causing the cycle, and the others are the ones that depend on it. In this case, the `ToyPlantRmModel` is the one causing the cycle, and the others are inter-dependent. We can read this as follows: - -1. `ToyPlantRmModel` depends on `ToyMaintenanceRespirationModel`, the plant-scale respiration sums up all organs respiration; -2. `ToyMaintenanceRespirationModel` depends on `ToyCBiomassModel`, the organs respiration depends on the organs biomass; -3. `ToyCBiomassModel` depends on `ToyCAllocationModel`, the organs biomass depends on the organs carbon allocation; -4. And finally `ToyCAllocationModel` depends on `ToyPlantRmModel` again, hence the cycle because the carbon allocation depends on the plant scale respiration. - -The models can not be ordered in a way that satisfies all dependencies, so the cycle can not be broken. To solve this issue, we need to re-think how models are mapped together, and break the cycle. - -There are several ways to break a cyclic dependency: - -- **Merge models**: If two models depend on each other because they need *e.g.* recursive computations, they can be merged into a third model that handles the computation and takes the two models as hard dependencies. Hard dependencies are models that are explicitly called by another model and do not participate on the building of the dependency graph. -- **Change models**: Of course models can be interchanged to avoid cyclic dependencies, but this is not really a solution, it is more a workaround. -- **PreviousTimeStep**: We can break the dependency graph by defining some variables as taken from the previous time step. A very well known example is the computation of the light interception by a plant that depends on the leaf area, which is usually the result of a model that also depends on the light interception. The cyclic dependency is usually broken by using the leaf area from the previous time step in the interception model, which is a good approximation for most cases. - -We can fix our previous mapping by computing the organs respiration using the carbon biomass from the previous time step instead. Let's see how to fix the cyclic dependency in our mapping (look at the leaf and internode scales): - -!!! details - ```@julia - mapping_nocyclic = Dict( - "Plant" => ( - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - :carbon_demand => ["Leaf", "Internode"], - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - MultiScaleModel( - model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], - ), - Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6, carbon_assimilation=5.0), - ), - "Internode" => ( - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - MultiScaleModel( - model=ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) - ), - Status(TT=10.0, carbon_biomass=1.0), - ), - "Leaf" => ( - ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - MultiScaleModel( - model=ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (second break) - ), - ToyCBiomassModel(1.2), - Status(TT=10.0), - ) - ); - nothing # hide - ``` - -The `ToyMaintenanceRespirationModel` models are now defined as `MultiScaleModel`, and the `carbon_biomass` variable is wrapped in a `PreviousTimeStep` structure. This structure tells PlantSimEngine to take the value of the variable from the previous time step, breaking the cyclic dependency. - -!!! note - `PreviousTimeStep` tells PlantSimEngine to take the value of the previous time step for the variable it wraps, or the value at initialization for the first time step. The value at initialization is the one provided by default in the models inputs, but is usually provided in the `Status` structure to override this default. - A `PreviousTimeStep` is used to wrap the **input** variable of a model, with or without a mapping to another scale *e.g.* `PreviousTimeStep(:carbon_biomass) => "Leaf"`. - -### Wrapping up - -In this section, we saw how to define a mapping between models and scales, run a simulation, and access the outputs. - -This is just a simple example, but PlantSimEngine can be used to define and combine much more complex models at multiple scales of detail. With its modular architecture and intuitive API, PlantSimEngine is a powerful tool for multi-scale plant growth and development modeling. diff --git a/docs/src/model_execution.md b/docs/src/model_execution.md index 2c261e763..d56ab6611 100644 --- a/docs/src/model_execution.md +++ b/docs/src/model_execution.md @@ -1,50 +1,10 @@ -# Model execution +# Model execution -## Simulation order +## Simulation order `PlantSimEngine.jl` uses the [`ModelList`](@ref) to automatically compute a dependency graph between the models and run the simulation in the correct order. When running a simulation with [`run!`](@ref), the models are then executed following this simple set of rules: -1. Independent models are run first. A model is independent if it can be run independently from other models, only using initializations (or nothing). +1. Independent models are run first. A model is independent if it can be run independently from other models, only using initializations (or nothing). 2. Then, models that have a dependency on other models are run. The first ones are the ones that depend on an independent model. Then the ones that are children of the second ones, and then their children ... until no children are found anymore. There are two types of children models (*i.e.* dependencies): hard and soft dependencies: - 1. Hard dependencies are always run before soft dependencies. A hard dependency is a model that list dependencies in their own method for `dep`. See [this example](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/3d91bb053ddbd087d38dcffcedd33a9db35a0fcc/examples/dummy.jl#L39) that shows `Process2Model` defining a hard dependency on any model that simulate `process1`. Inner hard dependency graphs (*i.e.* consecutive hard-dependency children) are considered as a single soft dependency. + 1. Hard dependencies are always run before soft dependencies. A hard dependency is a model that is directly called by another model. It is declared as such by its parent that lists its hard-dependencies as `dep`. See [this example](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/3d91bb053ddbd087d38dcffcedd33a9db35a0fcc/examples/dummy.jl#L39) that shows `Process2Model` defining a hard dependency on any model that simulates `process1`. 2. Soft dependencies are then run sequentially. A model has a soft dependency on another model if one or more of its inputs is computed by another model. If a soft dependency has several parent nodes (*e.g.* two different models compute two inputs of the model), it is run only if all its parent nodes have been run already. In practice, when we visit a node that has one of its parent that did not run already, we stop the visit of this branch. The node will eventually be visited from the branch of the last parent that was run. - -## Parallel execution - -### FLoops - -`PlantSimEngine.jl` uses the [`Floops`](https://juliafolds.github.io/FLoops.jl/stable/) package to run the simulation in sequential, parallel (multi-threaded) or distributed (multi-process) computations over objects, time-steps and independent processes. - -That means that you can provide any compatible executor to the `executor` argument of [`run!`](@ref). By default, [`run!`](@ref) uses the [`ThreadedEx`](https://juliafolds.github.io/FLoops.jl/stable/reference/api/#executor) executor, which is a multi-threaded executor. You can also use the [`SequentialEx`](https://juliafolds.github.io/Transducers.jl/dev/reference/manual/#Transducers.SequentialEx)for sequential execution (non-parallel), or [`DistributedEx`](https://juliafolds.github.io/Transducers.jl/dev/reference/manual/#Transducers.DistributedEx) for distributed computations. - -### Parallel traits - -`PlantSimEngine.jl` uses [Holy traits](https://invenia.github.io/blog/2019/11/06/julialang-features-part-2/) to define if a model can be run in parallel. - -!!! note - A model is executable in parallel over time-steps if it does not uses or set values from other time-steps, and over objects if it does not uses or set values from other objects. - -You can define a model as executable in parallel by defining the traits for time-steps and objects. For example, the `ToyLAIModel` model from the [examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/tree/main/examples) can be run in parallel over time-steps and objects, so it defines the following traits: - -```julia -PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyLAIModel}) = PlantSimEngine.IsTimeStepIndependent() -PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyLAIModel}) = PlantSimEngine.IsObjectIndependent() -``` - -By default all models are considered not executable in parallel, because it is the safest option to avoid bugs that are difficult to catch, so you only need to define these traits if it is executable in parallel for them. - -!!! tip - A model that is defined executable in parallel will not necessarily will. First, the user has to pass a parallel `executor` to [`run!`](@ref) (*e.g.* `ThreadedEx`). Second, if the model is coupled with another model that is not executable in parallel, `PlantSimEngine` will run all models in sequential. - -### Further executors - -You can also take a look at [FoldsThreads.jl](https://github.com/JuliaFolds/FoldsThreads.jl) for extra thread-based executors, [FoldsDagger.jl](https://github.com/JuliaFolds/FoldsDagger.jl) for -Transducers.jl-compatible parallel fold implemented using the Dagger.jl framework, and soon [FoldsCUDA.jl](https://github.com/JuliaFolds/FoldsCUDA.jl) for GPU computations -(see [this issue](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues/22)) and [FoldsKernelAbstractions.jl](https://github.com/JuliaFolds/FoldsKernelAbstractions.jl). You can also take a look at -[ParallelMagics.jl](https://github.com/JuliaFolds/ParallelMagics.jl) to check if automatic parallelization is possible. - -Finally, you can take a look into [Transducers.jl's documentation](https://github.com/JuliaFolds/Transducers.jl) for more information, for example if you don't know what is an executor, you can look into [this explanation](https://juliafolds.github.io/Transducers.jl/stable/explanation/glossary/#glossary-executor). - -## Tutorial - -You can learn how to run a simulation from [the home page](@ref PlantSimEngine), or from the [documentation of PlantBiophysics.jl](https://vezy.github.io/PlantBiophysics.jl/stable/simulation/first_simulation/). \ No newline at end of file diff --git a/docs/src/model_switching.md b/docs/src/model_switching.md deleted file mode 100644 index 765c736c8..000000000 --- a/docs/src/model_switching.md +++ /dev/null @@ -1,119 +0,0 @@ -# Model switching - -```@setup usepkg -using PlantSimEngine, PlantMeteo, CSV, DataFrames -# Import the examples defined in the `Examples` sub-module -using PlantSimEngine.Examples - -meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) - -models = ModelList( - ToyLAIModel(), - Beer(0.5), - ToyRUEGrowthModel(0.2), - status=(TT_cu=cumsum(meteo_day.TT),), -) -run!(models, meteo_day) -models2 = ModelList( - ToyLAIModel(), - Beer(0.5), - ToyAssimGrowthModel(), - status=(TT_cu=cumsum(meteo_day.TT),), -) -run!(models2, meteo_day) -``` - -One of the main objective of PlantSimEngine is allowing users to switch between model implementations for a given process **without making any change to the code**. - -The package was carefully designed around this idea to make it easy and computationally efficient. This is done by using the `ModelList`, which is used to list models, and the `run!` function to run the simulation following the dependency graph and leveraging Julia's multiple dispatch to run the models. - -## ModelList - -The `ModelList` is a container that holds a list of models, their parameter values, and the status of the variables associated to them. - -Model coupling is done by adding models to the `ModelList`. Let's create a `ModelList` with several models from the example scripts in the [`examples`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/) folder: - -Importing the models from the scripts: - -```julia -using PlantSimEngine -# Import the examples defined in the `Examples` sub-module: -using PlantSimEngine.Examples -``` - -Coupling the models in a `ModelList`: - -```@example usepkg -models = ModelList( - ToyLAIModel(), - Beer(0.5), - ToyRUEGrowthModel(0.2), - status=(TT_cu=cumsum(meteo_day.TT),), -) - -nothing # hide -``` - -PlantSimEngine uses the `ModelList` to compute the dependency graph of the models. Here we have seven models, one for each process. The dependency graph is computed automatically by PlantSimEngine, and is used to run the simulation in the correct order. - -We can run the simulation by calling the `run!` function with a meteorology. Here we use an example meteorology: - -```@example usepkg -meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) -nothing # hide -``` - -!!! tip - To reproduce this meteorology, you can check the code presented [in this section in the FAQ](@ref defining_the_meteo) - -We can now run the simulation: - -```@example usepkg -run!(models, meteo_day) -``` - -!!! note - You'll notice a warning returned by `run!` here. If you read its content, you'll see it says that `ToyRUEGrowthModel` does not allow for parallel computations over time-steps. This is because it uses values from the previous time-steps in its computations. By default, `run!` makes the simulations in parallel, so to avoid the warning, you must explicitly tell it to use a sequential execution instead. To do so, you can use the `executor=SequentialEx()` keyword argument. - -And then we can access the status of the `ModelList` using the [`status`](@ref) function: - -```@example usepkg -status(models) -``` - -Now what if we want to switch the model that computes growth ? We can do this by simply replacing the model in the `ModelList`, and PlantSimEngine will automatically update the dependency graph, and adapt the simulation to the new model. - -Let's switch `ToyRUEGrowthModel` by `ToyAssimGrowthModel`: - -```@example usepkg -models2 = ModelList( - ToyLAIModel(), - Beer(0.5), - ToyAssimGrowthModel(), # This was `ToyRUEGrowthModel(0.2)` before - status=(TT_cu=cumsum(meteo_day.TT),), -) - -nothing # hide -``` - -`ToyAssimGrowthModel` is a little bit more complex than `ToyRUEGrowthModel`, as it also computes the maintenance and growth respiration of the plant, so it has more parameters (we use the default values here). - -We can run a new simulation: - -```@example usepkg -run!(models2, meteo_day) -``` - -And we can see that the status of the variables is different from the previous simulation: - -```@example usepkg -status(models2) -``` - -!!! note - In our example we replaced a soft-dependency model, but the same principle applies to hard-dependency models. - -And that's it! We can switch between models without changing the code, and without having to recompute the dependency graph manually. This is a very powerful feature of PlantSimEngine!💪 - -!!! note - This was a very standard but easy example. Sometimes other models will require to add other models to the `ModelList`. For example `ToyAssimGrowthModel` could have required a maintenance respiration model. In this case `PlantSimEngine` will tell you that this kind of model is required for the simulation. \ No newline at end of file diff --git a/docs/src/multiscale/multiscale.md b/docs/src/multiscale/multiscale.md new file mode 100644 index 000000000..3c7948954 --- /dev/null +++ b/docs/src/multiscale/multiscale.md @@ -0,0 +1,255 @@ +# Multi-scale variable mapping + +The previous page showed how to convert a single-scale simulation to multi-scale. + +This page provides another example showcasing the nuances in variable mapping, with a more complex fully multiscale version of a prior simulation. The models will all be taken form the [examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/tree/main/examples). + +```@contents +Pages = ["multiscale.md"] +Depth = 3 +``` + +## Starting with a single-model mapping + +Let's import the `PlantSimEngine` package and all the example models we will use in this tutorial: + +```@example usepkg +using PlantSimEngine +using PlantSimEngine.Examples # Import some example models +``` + +Let's create a simple mapping with only one initial model, the carbon assimilation process ToyAssimModel, which will operate on leaves. +It resembles the ToyAssimGrowth model used in the single-scale simulation [Model switching](@ref) subsection. + +Our mapping between scale and model is therefore: + +```@example usepkg +mapping = Dict("Leaf" => ToyAssimModel()) +``` + +Just like in single-scale simulations, we can call `to_initialize` to check whether variables need to be initialised. It will this time index by scale: + +```@example usepkg +to_initialize(mapping) +``` + +In this example, the ToyAssimModel needs `:aPPFD` and `:soil_water_content` as inputs, which aren't initialised in our mapping. + +The initialization values for the variables can be passed along via a [`Status`](@ref) object: + +```@example usepkg +mapping = Dict( + "Leaf" => ( + ToyAssimModel(), + Status(aPPFD=1300.0, soil_water_content=0.5), + ), +) +``` + +If we call [`to_initialize`](@ref) on this new mapping, it returns an empty dictionary, meaning the mapping is valid, and we can start the simulation: + +```@example usepkg +to_initialize(mapping) +``` + +## Multiscale mapping between models and scales + +The `soil_water_content` variable was provided via the mapping. No model affects it, so it is constant in the above example. We could instead provide a model that computes it based on weather data, and/or a more realistic physical process. + +It also makes sense to have that model operate at a different scale than the "Leaf" scale. There is a dummy soil model called `ToySoilModel` in the examples folder. Let's put it at a new "Soil" scale level. + +ToyAssimModel is now makes use of the `soil_water_content` variable from the `"Soil"` scale, instead of at its own scale via the `Status` initialization. We therefore need to map `soil_water_content` from the "Soil" to the "Leaf" scale by wrapping `ToyAssimModel` in a `MultiScaleModel`: + +```@example usepkg +mapping = Dict( + "Soil" => ToySoilWaterModel(), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil" => :soil_water_content,], + ), + Status(aPPFD=1300.0), + ), +); +nothing # hide +``` + +In this example, we map the `soil_water_content` variable at scale "Leaf" to the `soil_water_content` variable at the `"Soil"` scale. If the name of the variable is the same between both scales, we can omit the variable name at the origin scale, *e.g.* `[:soil_water_content => "Soil"]`. + +The variable `aPPFD` is still provided in the `Status` type as a constant value. + +We can check again if the mapping is valid by calling [`to_initialize`](@ref): + +```@example usepkg +to_initialize(mapping) +``` + +Once again, `to_initialize` returns an empty dictionary, meaning the mapping is valid. + +## A more elaborate multiscale model mapping + +Let's now expand this mapping, to showcase other ways in which variables can be mapped from one scale to another. We'll keep the first two models, and add several more to simulate a couple of other processes within our plant. + +```@example usepkg +mapping = Dict( + "Scene" => ToyDegreeDaysCumulModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.6), + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + ), + ), + "Internode" => ( + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",], + ), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(carbon_biomass=1.0), + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil", :aPPFD => "Plant"], + ), + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",], + ), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + Status(carbon_biomass=0.5), + ), + "Soil" => ( + ToySoilWaterModel(), + ), +); +nothing # hide +``` + +This mapping might seem a little more daunting than previous examples, but several models should be recognizable in passing. In fact, you can consider this mapping to be an enhanced and more complex multi-scale version of a previous single-scale example, the coupling between photosynthesis model, a LAI model and a carbon biomass increment model, used in the [Model switching](@ref) subsection. + +```julia +models2 = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyAssimGrowthModel(), + status=(TT_cu=cumsum(meteo_day.TT),), +) +``` + +The multi-scale models simulate carbon capture via photosynthesis and carbon allocation for the plant organs' maintenance respiration and development. + +The LAI and photosynthesis models are the same as in the ModelList example. The [`ToyDegreeDaysCumulModel`](@ref) provides the Cumulative Thermal Time to the plant. + +The newly introduced models have the following dynamic : + +Carbon allocation is determined (ToyCAllocationModel) for the different organs of the plant (`"Leaf"` and `"Internode"`) from the assimilation at the `"Leaf"` scale (*i.e.* the offer) and their carbon demand (ToyCDemandModel). The `"Soil"` scale is used to compute the soil water content (`ToySoilWaterModel`](@ref)), which is needed to calculate the assimilation at the `"Leaf"` scale (ToyAssimModel). Also note that maintenance respiration at computed at the `"Leaf"` and `"Internode"` scales (ToyMaintenanceRespirationModel), and aggregated to compute the total maintenance respiration at the `"Plant"` scale (ToyPlantRmModel). + +## Different possible variable mappings + +The above mapping showcases the different ways to define how the variables are mapped in a `MultiScaleModel` : + +```julia + mapped_variables=[:TT_cu => "Scene",], +``` + +- At the "Plant" scale, the TT_cu variable is mapped as a scalar from the "Scene" scale. There is only a single "Scene" node in the MTG, and only a single "TT_cu" value per timestep for the simulation. + +```julia +:carbon_allocation => ["Leaf"] +``` + +- On the other hand, we have `:carbon_allocation => ["Leaf"]` at the plant scale for `ToyCAllocationModel`. The `carbon_assimilation` variable is mapped as a vector: there are multiple "Leaf" nodes, but only one "Plant" node, which aggregrates the value over every single leaf. This gives us a 'many-to-one' vector mapping, and in the [`run!`](@ref) functions for models at that scale `carbon_allocation` will be available in the `status` as a vector. + +```julia +:carbon_allocation => ["Leaf", "Internode"] +``` + +- A third type of the mapping would be `:carbon_allocation => ["Leaf", "Internode"]`, which provides values for a variable from several other scales simultaneously. In this case, the values are also available as a vector in the `carbon_assimilation` variable of the [`status`](@ref) inside the model, sorted in the same order as nodes are traversed in the graph. + +```julia +:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm] +``` + +- Finally, to map to a specific variable name at the target scale, *e.g.* `:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm]`. This syntax is useful when the variable name is different between scales, and we want to map to a specific variable name at the target scale. In this example, the variable `Rm_organs` at plant scale takes its values (is mapped) from the variable `Rm` at the `"Leaf"` and `"Internode"` scales. + +## Running a simulation + +Now that we have a valid mapping, we can run a simulation. Running a multiscale simulation requires a plant graph and the definition of the output variables we want dynamically for each scale. + +### Plant graph + +We can import an example multi-scale tree graph like so: + +```@example usepkg +mtg = import_mtg_example() +``` + +!!! note + You can use `import_mtg_example` only if you previously imported the `Examples` sub-module of PlantSimEngine, *i.e.* `using PlantSimEngine.Examples`. + +This graph has a root node that defines a scene, then a soil, and a plant with two internodes and two leaves. + +### Output variables + +For long simulations on plants with many organs, the output data can be very significant. It's possible to restrict the output variables that are tracked for the whole simulation to a subset of all the variables: + +```@example usepkg +outs = Dict( + "Scene" => (:TT, :TT_cu,), + "Plant" => (:aPPFD, :LAI), + "Leaf" => (:carbon_assimilation, :carbon_demand, :carbon_allocation, :TT), + "Internode" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), +) +``` + +This dictionary can be passed to the simulation via the optional `tracked_outputs` keyword argument to the [`run!`](@ref) function (see the next part). If no dictionary is provided, every variable will be tracked. + +These variables will be available in the output returned by [`run!`](@ref), with a value for each time step. The corresponding timestep and node in the MTG are also returned. + +### Meteorological data + +As for mono-scale models, we need to provide meteorological data to run a simulation. We can use the `PlantMeteo` package to generate some dummy data for two time steps: + +```@example usepkg +meteo = Weather( + [ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f = 200.0), + Atmosphere(T=25.0, Wind=0.5, Rh=0.8, Ri_PAR_f = 180.0) +] +) +``` + +### Simulation + +Let's make a simulation using the graph and outputs we just defined: + +```@example usepkg +outputs_sim = run!(mtg, mapping, meteo, tracked_outputs = outs); +nothing # hide +``` + +And that's it! We can now access the outputs for each scale as a dictionary of vectors of values per variable and scale. + +Or as a `DataFrame` using the [`DataFrames`](https://dataframes.juliadata.org) package: + +```@example usepkg +using DataFrames +convert_outputs(outputs_sim, DataFrame) +``` \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_considerations.md b/docs/src/multiscale/multiscale_considerations.md new file mode 100644 index 000000000..b2ddf2b14 --- /dev/null +++ b/docs/src/multiscale/multiscale_considerations.md @@ -0,0 +1,140 @@ +# Multi-scale considerations + +```@contents +Pages = ["multiscale_considerations.md"] +Depth = 3 +``` + +This page briefly details the subtle ways in which multi-scale simulations differ from prior single-scale simulations. The next few pages will showcase some of these subtleties with examples. + +Declaring and running a multi-scale simulation follows the same general workflow as the single-scale version, but multi-scale simulations do have some differences : + +- a simulation requires a Multi-scale Tree Graph (MTG) to run and operates on that graph +- when running, models are tied to a scale and only access local information +- models can run multiple times per timestep, +- the [`ModelList`](@ref) is replaced by a slightly more complex model mapping to link models to the scale they will operate at. + +The simulation dependency graph will still be computed automatically and handle most couplings, meaning users don't need to specify the order of model execution once the extra code to declare the models is written. You will still need to declare hard dependencies, with extra considerations for multi-scale hard dependencies. + +Multi-scale simulations also tend to require more extra ad hoc models to prepare some variables for some models. + +## Related pages + +Other pages in the multiscale section describe : + +- How to write a direct conversion of a single-scale ModelList simulation to a multi-scale simulation and add a second scale to it: [Converting a single-scale simulation to multi-scale](@ref), +- A more complex multi-scale version of the single-scale simulation showcasing different variable mappings between scales: [Multi-scale variable mapping](@ref), +- A three-part tutorial describing how to build up a combination of models to simulate a growing toy plant: [Writing a multiscale simulation](@ref), +- Ways to handle situations where a variable ends up causing a cyclic dependency: [Avoiding cyclic dependencies](@ref), +- Multi-scale specific coupling considerations and subtleties:[Handling dependencies in a multiscale context](@ref) + +## Multi-scale tree graphs + +Functional-Structural Plant Models are often about simulating plant growth. A multi-scale simulation is implicitely expected to operate on a plant-like object, represented by a multi-scale tree graph. + +A multi-scale tree graph (MTG) object (see the [Multi-scale Tree Graphs](@ref) subsection for a quick description) is therefore required to run a multi-scale simulations. It can be a dummy MTG if the simulation doesn't actually affect it, but is nevertheless a required argument to the multi-scale [`run!`](@ref) function. + +All the multi-scale examples make use of the companion package [MultiScaleTreeGraph.jl](https://github.com/VEZY/MultiScaleTreeGraph.jl), which we therefore recommend for running your own multi-scale simulations. Visualizing a Multi-scale Tree Graph can be done using [PlantGeom](https://github.com/VEZY/PlantGeom.jl). + +!!! note + Multi-scale Tree Graphs make use of conflicting terminology with PlantSimEngine's concepts, which is discussed in [Scale/symbol terminology ambiguity](@ref). If you are new to those concepts, make sure to read that section and keep note of it. + +## Models run once per organ instance, not once per organ level + +Some models, like the ones we've seen in single-scale simulations, work on a very simple model of a whole plant. + +More fine-grained models can be tied to a specific plant organ. + +For instance, a model computing a leaf's surface area depending on its age would operate at the "leaf" scale, and be called **for every leaf** at every timestep. On the other hand, a model computing the plant's total leaf area only needs to be run once per timestep, and can be run at the "Plant" scale. + +This is a major difference between a single-scale simulation and a multi-scale one. By default, any model in a single-scale simulation will only run **once** per timestep. However, in multi-scale, if a plant has several instances of an organ type -say it has a hundred leaves- then any model operating at the "Leaf" scale will by default run one hundred times per timestep, unless it is explicitely controlled by another model (which can happen in hard dependency configurations). + +## Mappings + +When users define which models they use, PlantSimEngine cannot determine in advance which scale level they operate at. This is partly because the plant organs in an MTG do not have standardized names, and partly because some plant organs might not be part of the initial MTG, so parsing it isn't enough to infer what scales are used. + +The user therefore needs to indicate for a simulation's which models are related to which scale. + +A multi-scale mapping links models to the scale at which they operate, and is implemented as a Julia `Dict`, tying a scale, such as "Leaf" to models operating at that scale, such as "LeafSurfaceAreaModel". It is the equivalent of a [`ModelList`](@ref) in a single-scale simulation. + +Multi-scale models can be similar models to the ones found in earlier sections, or, if they need to make use of variables at other scales, may need to be wrapped as part of a [`MultiScaleModel`](@ref) object. Many models are not tied to a particular scale, which means those models can be reused at different scales or in single-scale simulations. + +## The simulation operates on an MTG + +Unlike in single-scale simulations, which make use of a [`Status`](@ref) object to store the current state of every variable in a simulation, multi-scale simulations operate on a per-organ basis. + +This means every organ instance has its own [`Status`](@ref), with scale-specific attributes. + +This has two **important** consequences in terms of running a simulation : + +- First, **any scale absent from the MTG will not be run**. If your MTG contains no leaves, then no model operating at the scale "Leaf" will be able to run until a "Leaf" organ is created and a node is added in the MTG. Otherwise, it has no MTG node to operate on. The only exceptions are hard dependency models which can be called from a different scale, since they can be called directly by a model on a node at a different existing scale, even if there is no node at their own scale. + +- Secondly, models only have access to **local** organ information. The [`status`](@ref) argument in the [`run!`](@ref) function only contains variables **at the model's scale**, unless variables from other scales are mapped via a [`MultiScaleModel`](@ref) wrapping. + +## The run! function's signature + +The [`run!`](@ref) function differs slightly from its single-scale version. The current structure (excluding a couple of advanced/deprecated kwargs) is the following: + +```julia +run!(mtg, mapping, meteo, constants, extra; nsteps, tracked_outputs) +``` + +Instead of a [`ModelList`](@ref), it takes an MTG and a mapping. The optional `meteo` and `constants` argument are identical to the single-scale version. The `extra` argument is now reserved and should not be used. A new `nsteps` keyword argument is available to restrict the simulation to a specified number of steps. + +## Multi-scale output data structure + +The output structure, like the mapping, is a Julia `Dict` structure indexed by scale. In each scale, another `Dict` maps variables to their values per timestep, per node. This makes the structure a little bulkier and a little more verbose to inspect than in single-scale, but the general usage is similar. Multiscale Tree Graph nodes are also added to the output data, as a `:node` entry. + +To illustrate, here's an example output from part 3 of the Toy plant tutorial, zeroing in on a variable at the "Root" scale: [Fixing bugs in the plant simulation](@ref): + +```julia +julia> outs + +Dict{String, Dict{Symbol, Vector}} with 5 entries: + "Internode" => Dict(:carbon_root_creation_consumed=>[[50.0, 50.0], [50.0, 50.0], [50.0, 50.0], [50.0, 50.0], [50.0, … + "Root" => Dict(:carbon_root_creation_consumed=>[[50.0, 50.0], [50.0, 50.0, 50.0], [50.0, 50.0, 50.0, 50.0], [50… + "Scene" => Dict(:TT_cu=>[[0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0] … [2099.61], [20… + "Plant" => Dict(:carbon_root_creation_consumed=>[[50.0], [50.0], [50.0], [50.0], [50.0], [50.0], [50.0], [50.0],… + "Leaf" => Dict(:node=>Vector{Node{NodeMTG, Dict{Symbol, Any}}}[[+ 4: Leaf… + +julia> outs["Root"] +Dict{Symbol, Vector} with 4 entries: + :carbon_root_creation_consumed => [[50.0, 50.0], [50.0, 50.0, 50.0], [50.0, 50.0, 50.0, 50.0], [50.0, 50.0, 50.0, 50… + :node => Vector{Node{NodeMTG, Dict{Symbol, Any}}}[[+ 9: Root… + :water_absorbed => [[0.5, 0.0], [1.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0], [1.1, 1.1, 1.1, 1.1, 0.0], [0.… + :root_water_assimilation => [[1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0], [1.0, 1.0, 1.0, 1.0, 1.0], [1.… + +julia> outs["Root"][:carbon_root_creation_consumed] +365-element Vector{Vector{Float64}}: + [50.0, 50.0] # timestep 1: two root nodes + [50.0, 50.0, 50.0] + [50.0, 50.0, 50.0, 50.0] + [50.0, 50.0, 50.0, 50.0, 50.0] + [50.0, 50.0, 50.0, 50.0, 50.0, 50.0] + [50.0, 50.0, 50.0, 50.0, 50.0, 50.0, 50.0] # timestep 6: 7 root nodes + ⋮ +``` + +As more roots get added in this simulation, the vectors expand to list the values of all the nodes for every variable for every timestep. + +!!! warning + Currently, the `:node` entry only shallow copies nodes. The `:node` values at each scale for every timestep actually reflect the final state of the node, meaning attribute values may not correspond to the value at that timestep. You may need to output these values via a dedicated model to keep track of them properly. + Also note that there currently is no way of removing nodes. Nodes corresponding to organs considered to be pruned/dead/aborted are still present in the output data structure. + +Multi-scale simulations, especially for plants which have thousands of leaves, internodes, root branches, buds and fruits, may compute huge amounts of data. Just like in single-scale simulations, it is possible to keep only variables whose values you want to track for every timestep, and filter the rest out, using the `tracked_outputs` keyword argument for the [`run!`](@ref) function. + +Those tracked variables also need to be indexed by scale to avoid ambiguity: + +```julia +outs = Dict( + "Scene" => (:TT, :TT_cu,), + "Plant" => (:aPPFD, :LAI), + "Leaf" => (:carbon_assimilation, :carbon_demand, :carbon_allocation, :TT), + "Internode" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), +) +``` + +## Coupling and multi-scale hard dependencies + +Multi-scale brings new types of coupling: mappings are part of the approach used to handle variables used by models at different scales. A model can also have a hard dependency on another model that operates at another scale. This multi-scale-specific complexity is discussed in [Handling dependencies in a multiscale context](@ref) \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_coupling.md b/docs/src/multiscale/multiscale_coupling.md new file mode 100644 index 000000000..66934e88a --- /dev/null +++ b/docs/src/multiscale/multiscale_coupling.md @@ -0,0 +1,171 @@ + +# Handling dependencies in a multiscale context + +```@contents +Pages = ["multiscale_coupling.md"] +Depth = 3 +``` + +## Scalar and vector variable mappings + +In the detailed example discussed previously [Multi-scale variable mapping](@ref), there were several instances of mapping a variable from one scale to another, which we'll briefly describe again to help transition to the next and more advanced subsection. Here's a relevant exerpt from the mapping : + +```julia +"Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + ... + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + ... + ), +``` + +For flexibility reasons, instead of explicitely linking most models from different scales together, one only declares which variables are meant to be taken from another scale (or more accurately, a model at a different scale outputting those variables). This keeps the convenience of switching models while making few changes to the mapping. + +However, PlantSimEngine cannot infer which scales have multiple instances, and which are single-instance, as the scale names are user-defined. + +In the above example, there is only one scene at the "Scene", and one plant at the "Plant" scale, meaning the `TT_cu` variable mapped between the two has a one-to-one scalar-to-scalar correspondance. + +On the other hand, the `carbon_assimilation` variable is computed for **every** leaf, of which there could be hundreds, or thousands, giving a scalar-to-vector correspondance. The carbon assimilation model runs many times every timestep, whereas the carbon allocation model only runs once per timestep. There may be initially be only a single leaf, though, meaning PlantSimEngine cannot currently guess from the initial configuration that there might be multiple leaves created during the simulation. + +Hence the difference in mapping declaration : `TT_cu`is declared as a scalar correspondence : +```julia +:TT_cu => "Scene", +``` +whereas `carbon_assimilation` (and other variables) will be declared as a vector correspondence : +```julia +:carbon_assimilation => ["Leaf"], +``` + +Note that there may be instances where you might wish to write your own model to aggregate a variable from a multi-instance scale. + +## Hard dependencies between models at different scale levels + +If a model requires some input variable that is computed at another scale, then providing the appropriate mapping for that variable will resolve name conflicts and enable that model to run with no further steps for the user or the modeler when the coupling is a 'soft dependency'. + +In the case of a hard dependency that operates **at the same scale as its parent**, declaring the hard dependency is exactly the same as in single-scale simulations and there are also no new extra steps on the user-side: + +- The parent model directly handles the call to its hard dependency model(s), meaning they are not explicitely managed by the top-level dependency graph. +- This means only the owning model of that dependency is visible in the graph, and its hard dependency nodes are internal. +- When the caller (or any downstream model that requires some variables from the hard dependency model) operates at the same scale, variables are easily accessible, and no mapping is required. + +On the other hand, modelers do need to bear in mind a couple of subtleties when developing models that possess hard dependencies that operate **at a different organ level from their parent**: + +If an model needs to be directly called by a parent but operates at a different scale/organ level, a modeler must declare hard dependencies with their respective organ level, similarly to the way the user provides a mapping. + +Conceptually : + +```julia + PlantSimEngine.dep(m::ParentModel) = ( + name_provided_in_the_mapping=AbstractHardDependencyModel => ["Organ_Name_1",], +) +``` + +### An example from the toy plant simulation tutorial + +You can find an example of a hard dependency discussed in the [A multi-scale hard dependency appears](@ref) subsection of the third part of toy plant tutorial. + +### An example from XPalm.jl + +Here's a concrete example in [XPalm](https://github.com/PalmStudio/XPalm.jl), an oil palm model developed on top of PlantSimEngine. + Organs are produced at the phytomer scale, but need to run an age model and a biomass model at the reproductive organs' scales. + +```julia + PlantSimEngine.dep(m::ReproductiveOrganEmission) = ( + initiation_age=AbstractInitiation_AgeModel => [m.male_symbol, m.female_symbol], + final_potential_biomass=AbstractFinal_Potential_BiomassModel => [m.male_symbol, m.female_symbol], +) +``` + +The user-mapping includes the required models at specific organ levels. Here's the relevant portion of the mapping for the male reproductive organ : + +```julia +mapping = Dict( + ... + "Male" => + MultiScaleModel( + model=XPalm.InitiationAgeFromPlantAge(), + mapped_variables=[:plant_age => "Plant",], + ), + ... + XPalm.MaleFinalPotentialBiomass( + p.parameters[:male][:male_max_biomass], + p.parameters[:male][:age_mature_male], + p.parameters[:male][:fraction_biomass_first_male], + ), + ... +) +``` + +The model's constructor provides convenient default names for the scale corresponding to the reproductive organs. A user may override that if their naming schemes or MTG attributes differ. + +```julia +function ReproductiveOrganEmission(mtg::MultiScaleTreeGraph.Node; phytomer_symbol="Phytomer", male_symbol="Male", female_symbol="Female") + ... +end +``` + +## Implementation details: accessing a hard dependency's variables from a different scale + +But how does a model M calling a hard dependency H provide H's variables when calling H's [`run!`](@ref) function ? The [`status`](@ref) argument the user provides M operates at M's organ level, so if used to call H's run! function any required variable for H will be missing. + +PlantSimEngine provides what are called Status Templates in the simulation graph. Each organ level has its own Status template listing the available variables at that scale. +So when a model M calls a hard dependency H's [`run!`](@ref) function, any required variables can be accessed through the status template of H's organ level. + +### Back to the XPalm example + +Using the same example in XPalm, the oil palm FSPM: + +```julia +# Note that the function's 'status' parameter does NOT contain the variables required by the hard dependencies as the calling model's organ level is "Phytomer", not "Male" or "Female" + +function PlantSimEngine.run!(m::ReproductiveOrganEmission, models, status, meteo, constants, sim_object) + ... + status.graph_node_count += 1 + + # Create the new organ as a child of the phytomer: + st_repro_organ = add_organ!( + status.node[1], # The phytomer's internode is its first child + sim_object, # The simulation object, so we can add the new status + "+", status.sex, 4; + index=status.phytomer_count, + id=status.graph_node_count, + attributes=Dict{Symbol,Any}() + ) + + # Compute the initiation age of the organ: + PlantSimEngine.run!(sim_object.models[status.sex].initiation_age, sim_object.models[status.sex], st_repro_organ, meteo, constants, sim_object) + PlantSimEngine.run!(sim_object.models[status.sex].final_potential_biomass, sim_object.models[status.sex], st_repro_organ, meteo, constants, sim_object) +end +``` + +In the above example the organ and its status template are created on the fly. +When that isn't the case, the status template can be accessed through the simulation graph : + +```julia +function PlantSimEngine.run!(m::ReproductiveOrganEmission, models, status, meteo, constants, sim_object) + + ... + + if status.sex == "Male" + + status_male = sim_object.statuses["Male"][1] + run!(sim_object.models["Male"].initiation_age, models, status_male, meteo, constants, sim_object) + run!(sim_object.models["Male"].final_potential_biomass, models, status_male, meteo, constants, sim_object) + else + # Female + ... + end +end +``` \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_cyclic.md b/docs/src/multiscale/multiscale_cyclic.md new file mode 100644 index 000000000..30943188c --- /dev/null +++ b/docs/src/multiscale/multiscale_cyclic.md @@ -0,0 +1,115 @@ +# Avoiding cyclic dependencies + +When defining a mapping between models and scales, it is important to avoid cyclic dependencies. A cyclic dependency occurs when a model at a given scale depends on a model at another scale that depends on the first model. Cyclic dependencies are bad because they lead to an infinite loop in the simulation (the dependency graph keeps cycling indefinitely). + +PlantSimEngine will detect cyclic dependencies and raise an error if one is found. The error message indicates the models involved in the cycle, and the model that is causing the cycle will be highlighted in red. + +For example the following mapping will raise an error: + +!!! details + Example mapping + + ```julia + mapping_cyclic = Dict( + "Plant" => ( + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + ), + Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6), + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(TT=10.0, carbon_biomass=1.0), + ), + "Leaf" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + ToyCBiomassModel(1.2), + Status(TT=10.0), + ) + ) + ``` + +Let's see what happens when we try to build the dependency graph for this mapping: + +```julia +julia> dep(mapping_cyclic) +ERROR: Cyclic dependency detected in the graph. Cycle: + Plant: ToyPlantRmModel + └ Leaf: ToyMaintenanceRespirationModel + └ Leaf: ToyCBiomassModel + └ Plant: ToyCAllocationModel + └ Plant: ToyPlantRmModel + + You can break the cycle using the `PreviousTimeStep` variable in the mapping. +``` + +How can we interpret the message? We have a list of five models involved in the cycle. The first model is the one causing the cycle, and the others are the ones that depend on it. In this case, the `ToyPlantRmModel` is the one causing the cycle, and the others are inter-dependent. We can read this as follows: + +1. `ToyPlantRmModel` depends on `ToyMaintenanceRespirationModel`, the plant-scale respiration sums up all organs respiration; +2. `ToyMaintenanceRespirationModel` depends on `ToyCBiomassModel`, the organs respiration depends on the organs biomass; +3. `ToyCBiomassModel` depends on `ToyCAllocationModel`, the organs biomass depends on the organs carbon allocation; +4. And finally `ToyCAllocationModel` depends on `ToyPlantRmModel` again, hence the cycle because the carbon allocation depends on the plant scale respiration. + +The models can not be ordered in a way that satisfies all dependencies, so the cycle can not be broken. To solve this issue, we need to re-think how models are mapped together, and break the cycle. + +There are several ways to break a cyclic dependency: + +- **Merge models**: If two models depend on each other because they need *e.g.* recursive computations, they can be merged into a third model that handles the computation and takes the two models as hard dependencies. Hard dependencies are models that are explicitly called by another model and do not participate on the building of the dependency graph. +- **Change models**: Of course models can be interchanged to avoid cyclic dependencies, but this is not really a solution, it is more a workaround. +- **PreviousTimeStep**: We can break the dependency graph by defining some variables as taken from the previous time step. A very well known example is the computation of the light interception by a plant that depends on the leaf area, which is usually the result of a model that also depends on the light interception. The cyclic dependency is usually broken by using the leaf area from the previous time step in the interception model, which is a good approximation for most cases. + +We can fix our previous mapping by computing the organs respiration using the carbon biomass from the previous time step instead. Let's see how to fix the cyclic dependency in our mapping (look at the leaf and internode scales): + +!!! details + ```@julia + mapping_nocyclic = Dict( + "Plant" => ( + MultiScaleModel( + model=ToyCAllocationModel(), + mapping=[ + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + ), + Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6, carbon_assimilation=5.0), + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + MultiScaleModel( + model=ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) + ), + Status(TT=10.0, carbon_biomass=1.0), + ), + "Leaf" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + MultiScaleModel( + model=ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (second break) + ), + ToyCBiomassModel(1.2), + Status(TT=10.0), + ) + ); + nothing # hide + ``` + +The `ToyMaintenanceRespirationModel` models are now defined as [`MultiScaleModel`](@ref), and the `carbon_biomass` variable is wrapped in a `PreviousTimeStep` structure. This structure tells PlantSimEngine to take the value of the variable from the previous time step, breaking the cyclic dependency. + +!!! note + [`PreviousTimeStep`](@ref) tells PlantSimEngine to take the value of the previous time step for the variable it wraps, or the value at initialization for the first time step. The value at initialization is the one provided by default in the models inputs, but is usually provided in the [`Status`](@ref) structure to override this default. + A [`PreviousTimeStep`](@ref) is used to wrap the **input** variable of a model, with or without a mapping to another scale *e.g.* `PreviousTimeStep(:carbon_biomass) => "Leaf"`. \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_example_1.md b/docs/src/multiscale/multiscale_example_1.md new file mode 100644 index 000000000..23e76bb83 --- /dev/null +++ b/docs/src/multiscale/multiscale_example_1.md @@ -0,0 +1,290 @@ +# Writing a multiscale simulation + +This three-part subsection walks you through building a multi-scale simulation from scratch. It is meant as an illustration of the iterative process you might go through when building and slowly tuning a Functional-Structural Plant Model, where previous multi-scale examples focused more on the API syntax. + +You can find the full script for the first part's toy simulation in the [ToyMultiScalePlantModel](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyMultiScalePlantModel/ToyPlantSimulation1.jl) subfolder of the examples folder. + +```@contents +Pages = ["multiscale_example_1.md"] +Depth = 3 +``` + +## Disclaimer + +The actual plant being created, as well as some of the custom models, have no real physical meaning and are very much ad hoc (which is why most of them aren't standalone in the examples folder). Similarly, some of the parameter values are pulled out of thin air, and have no ties to research papers or data. + +The main purpose here is to showcase PlantSimEngine's multi-scale features and how to structure your models, not accuracy, realism or performance. + +## Initial setup + +We'll need to make use of a few packages, as usual, after adding them to our Julia environment: + +```@example usepkg +using PlantSimEngine +using PlantSimEngine.Examples # to import the ToyDegreeDaysCumulModel model +using PlantMeteo +using MultiScaleTreeGraph # multi-scale +using CSV, DataFrames # used to import the example weather data +``` + +## A basic growing plant + +At minimum, to simulate some kind of fake growth, we need : + +- A Multi-scale Tree Graph representing the plant +- Some way of adding organs to the plant +- Some kind of temporality to spread this growth over multiple timesteps + +Let's have some concept of 'leaves' that capture the (carbon) resource necessary for organ growth, and let's have the organ emergence happen at the 'internode' level, to illustrate multiple organs with different behavior. + +We'll make the assumption that the internodes make use of carbon from a common pool. We'll also make use of thermal time as a growth delay factor. + +To sum up, we have: +- a MTG with growing internodes and leaves +- Individual leaves that capture carbon fed into a common pool +- Internodes which take from that pool to create new organs, with a thermal time constraint. + +One way of modeling this approach translates into several scales and models: + +- a Scene scale, for thermal time. The [`ToyDegreeDaysCumulModel`](@ref) from the [examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyDegreeDays.jl) provides thermal time from temperature data +- a Plant scale, where we'll define the carbon pool +- an Internode scale, which draws from the pool to create new organs +- a Leaf scale, which captures carbon + +Let's also add a very artificial limiting factor: if the total leaf surface area is above a threshold no new organs are created. + +We can expect the simulation mapping to look like a more complex version of the following: + +```julia +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ToyStockComputationModel(), +"Internode" => ToyCustomInternodeEmergence(), +"Leaf" => ToyLeafCarbonCaptureModel(), +) +``` + +Some of the models will need to gather variables from scales other than their own, meaning they will need to be converted into MultiScaleModels. + +## Implementation + +### Carbon Capture + +Let's start with the simplest model. Our fake leaves will continuously capture some constant amount of carbon every timestep. No inputs or parameters are required. + +```@example usepkg +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple() # No inputs +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + status.carbon_captured = 40 +end +``` + +### Resource storage + +The model storing resources for the whole plant needs a couple of inputs: the amount of carbon captured by the leaves, as well as the amount consumed by the creation of new organs. It outputs the current stock. + +```@example usepkg +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(carbon_captured=0.0,carbon_organ_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (carbon_stock=-Inf,) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) +end +``` + +### Organ creation + +This model is a modified version of the ToyInternodeEmergence model found [in the examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyInternodeEmergence.jl). An internode produces two leaves and a new internode. + +Let's first define a helper function that iterates across a Multiscale Tree Graph and returns the number of leaves : + +```@example usepkg +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end +``` + +Now that we have that, let's define a few parameters to the model. It requires : +- a thermal time emergence threshold +- a carbon cost for organ creation + +We'll also add a couple of other parameters, which could go elsewhere : +- the surface area of a leaf (no variation, no growth stages) +- the max leaf surface area beyond which organ creation stops + +```@example usepkg +PlantSimEngine.@process "organ_emergence" verbose = false + +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T +end +``` + +!!! note + We make use of parametric types instead of the intuitive Float64 for flexibility. See [Parametric types](@ref) for a more in-depth explanation + +And give them some default values : + +```@example usepkg +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0, leaves_max_surface_area=100.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area) +``` + +Our internode model requires thermal time, and the amount of available carbon, and outputs the amount of carbon consumed, as well as the last thermal time where emergence happened (this is useful when new organs can be produced multiple times, which won't be the case here). + +```@example usepkg +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0, carbon_stock=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) +``` +Finally, the [`run!`](@ref) function checks that conditions are met for new organ creation : +- thermal time threshold exceeded +- total leaf surface area not above limit +- carbon available +- no organs already created by that internode + +and then updates the MTG. + +```@example usepkg +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if not enough carbon, no organ creation + if status.carbon_stock < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end +``` + +### Updated mapping + +We can now define the final mapping for this simulation. + +The carbon capture and thermal time models don't need to be changed from the earlier version. +The organ creation model at the "Internode" scale needs the carbon stock from the "Plant" scale, as well as thermal time from the "Scene" scale. +The resource storing model at the "Plant" scale needs the carbon captured by **every** leaf, and the carbon consumed by **every** internode that created new organs this timestep. This requires mapping vector variables : + +```julia + mapped_variables=[ + :carbon_captured=>["Leaf"], + :carbon_organ_creation_consumed=>["Internode"] + ], +``` +as opposed to the single-valued carbon stock mapped variable : + +```julia + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:carbon_stock)=>"Plant"], +``` + +And of course, some variables need to be initialized in the status: + +```@example usepkg +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :carbon_organ_creation_consumed=>["Internode"] + ], + ), + Status(carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:carbon_stock)=>"Plant"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Leaf" => ToyLeafCarbonCaptureModel(), +) +``` + +!!! note + This excerpt (and the complete script file) showcase the final properly initialized mapping, but when developing, you are encouraged to make liberal use of the helper function [`to_initialize`](@ref) and check the PlantSimEngine user errors. + +### Running a simulation + +We only need an MTG, and some weather data, and then we'll be set. Let's create a simple MTG : + +```@example usepkg + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) +``` + +Import some weather data : + +```@example usepkg +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +nothing # hide +``` + +And we're good to go ! + +```@example usepkg +outs = run!(mtg, mapping, meteo_day) +``` + +If you query or display the MTG after simulation, you'll see it expanded and grew multiple internodes and leaves : + +```@example usepkg +mtg +#get_n_leaves(mtg) +``` + +And that's it ! Feel free to tinker with the parameters and see when things break down, to get a feel for the simulation. + +Of course, this is a very crude and unrealistic simulation, with many dubious assumptions and parameters. But significantly more complex modelling is possible using the same approach : XPalm runs using a few dozen models spread out over nine scales. + +This is a three-part tutorial and continues in the [Expanding on the multiscale simulation](@ref) page. \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_example_2.md b/docs/src/multiscale/multiscale_example_2.md new file mode 100644 index 000000000..4b33bb5e5 --- /dev/null +++ b/docs/src/multiscale/multiscale_example_2.md @@ -0,0 +1,267 @@ +# Expanding on the multiscale simulation + +Let's build on the previous example and add some other organ growth, as well as some very mild coupling between the two. + +You can find the full script for this simulation in the [ToyMultiScalePlantModel](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyMultiScalePlantModel/ToyPlantSimulation2.jl) subfolder of the examples folder. + +```@contents +Pages = ["multiscale_example_2.md"] +Depth = 3 +``` + +## Setup + +Once again, with a properly set-up Julia environment: + +```@example usepkg +using PlantSimEngine +using PlantSimEngine.Examples +using PlantMeteo +using MultiScaleTreeGraph +using CSV, DataFrames + +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple() # No inputs +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + status.carbon_captured = 40 +end + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end +``` + +## Adding roots to our plant + +We'll add a root that extracts water and adds it to the stock. Initial water stocks are low, so root growth is prioritized, then the plant also grows leaves and a new internode like it did before. Roots only grow up to a certain point, and don't branch. + +This leads to adding a new scale, "Root" to the mapping, as well as two more models, one for water absorption, the other for root growth. Other models are updated here and there to account for water. The carbon capture model remains unchanged, and so is the `get_n_leaves` helper function. + +## Root models + +### Water absorption + +Let's implement a very fake model of root water absorption. It'll capture the amount of precipitation in the weather data multiplied by some assimilation factor. + +```@example usepkg +PlantSimEngine.@process "water_absorption" verbose = false + +struct ToyWaterAbsorptionModel <: AbstractWater_AbsorptionModel +end + +PlantSimEngine.inputs_(::ToyWaterAbsorptionModel) = (root_water_assimilation=1.0,) +PlantSimEngine.outputs_(::ToyWaterAbsorptionModel) = (water_absorbed=0.0,) + +function PlantSimEngine.run!(m::ToyWaterAbsorptionModel, models, status, meteo, constants=nothing, extra=nothing) + status.water_absorbed = meteo.Precipitations * status.root_water_assimilation +end +``` + +### Root growth + +The root growth model is similar to the internode growth one : it checks for a water threshold and that there is enough carbon, and adds a new organ to the MTG if the maximum length hasn't been reached. + +It also makes use of a couple of helper functions to find the end root and compute root length : + +```@example usepkg +function get_root_end_node(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return MultiScaleTreeGraph.traverse(root, x->x, symbol="Root", filter_fun = MultiScaleTreeGraph.isleaf) +end + +function get_roots_count(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x->x, symbol="Root")) +end + +PlantSimEngine.@process "root_growth" verbose = false + +struct ToyRootGrowthModel{T} <: AbstractRoot_GrowthModel + water_threshold::T + carbon_root_creation_cost::T + root_max_len::Int +end + +PlantSimEngine.inputs_(::ToyRootGrowthModel) = (water_stock=0.0,carbon_stock=0.0,) +PlantSimEngine.outputs_(::ToyRootGrowthModel) = (carbon_root_creation_consumed=0.0,) + +function PlantSimEngine.run!(m::ToyRootGrowthModel, models, status, meteo, constants=nothing, extra=nothing) + if status.water_stock < m.water_threshold && status.carbon_stock > m.carbon_root_creation_cost + + root_end = get_root_end_node(status.node) + + if length(root_end) != 1 + throw(AssertionError("Couldn't find MTG leaf node with symbol \"Root\"")) + end + root_len = get_roots_count(root_end[1]) + if root_len < m.root_max_len + st = add_organ!(root_end[1], extra, "<", "Root", 2, index=1) + status.carbon_root_creation_consumed = m.carbon_root_creation_cost + end + else + status.carbon_root_creation_consumed = 0.0 + end +end +``` + +## Updating other models to account for water + +### Resource storage + +Water absorbed must now be accumulated, and root carbon creation costs taken into account. + +```@example usepkg +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(water_absorbed=0.0,carbon_captured=0.0,carbon_organ_creation_consumed=0.0,carbon_root_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (water_stock=-Inf,carbon_stock=-Inf) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.water_stock += sum(status.water_absorbed) + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) - sum(status.carbon_root_creation_consumed) +end +``` + +### Internode creation + +The minor change is that new organs are now created only if the water stock is above a given threshold. + +```@example usepkg +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T + water_leaf_threshold::T +end + +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0,leaves_max_surface_area=100.0, +water_leaf_threshold=30.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area, water_leaf_threshold) + +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0,water_stock=0.0, carbon_stock=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) + +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if water levels are low, prioritise roots + if status.water_stock < m.water_leaf_threshold + return nothing + end + + # if not enough carbon, no organ creation + if status.carbon_stock < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end +``` + +## Updating the mapping + +The resource storage and internode emergence models now need a couple of extra water-related mapped variables. +The "Root" organ is added to the mapping with its own models. New parameters need to be initialized. + +```@example usepkg +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + :carbon_root_creation_consumed=>["Root"], + :carbon_organ_creation_consumed=>["Internode"] + + ], + ), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:water_stock)=>"Plant", + PreviousTimeStep(:carbon_stock)=>"Plant"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Root" => ( MultiScaleModel( + model=ToyRootGrowthModel(10.0, 50.0, 10), + mapped_variables=[PreviousTimeStep(:carbon_stock)=>"Plant", + PreviousTimeStep(:water_stock)=>"Plant"], + ), + ToyWaterAbsorptionModel(), + Status(carbon_root_creation_consumed=0.0, root_water_assimilation=1.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) +``` + +## Running the simulation + +Running this new simulation is almost the same as before. The weather data is unchanged, but a new "Root" node was added to the MTG. + +```@example usepkg +mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + plant_root_start = MultiScaleTreeGraph.Node( + plant, + MultiScaleTreeGraph.NodeMTG("+", "Root", 1, 3), + ) + +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +outs = run!(mtg, mapping, meteo_day) +mtg +``` + +And that's it ! + +...Or is it ? + +If you inspect the code and output data closely, you may notice some distinctive problems with the way the simulation runs... Some things aren't quite right. If you wish to know more, onwards to the next chapter: [Fixing bugs in the plant simulation](@ref) \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_example_3.md b/docs/src/multiscale/multiscale_example_3.md new file mode 100644 index 000000000..749914071 --- /dev/null +++ b/docs/src/multiscale/multiscale_example_3.md @@ -0,0 +1,497 @@ +# Fixing bugs in the plant simulation + +```@setup usepkg +using PlantSimEngine +using PlantSimEngine.Examples +using PlantMeteo, CSV, DataFrames +using MultiScaleTreeGraph +function get_root_end_node(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return MultiScaleTreeGraph.traverse(root, x->x, symbol="Root", filter_fun = MultiScaleTreeGraph.isleaf) +end + +function get_roots_count(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x->x, symbol="Root")) +end + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end + +PlantSimEngine.@process "organ_emergence" verbose = false + +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T + water_leaf_threshold::T +end + +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0,leaves_max_surface_area=100.0, +water_leaf_threshold=30.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area, water_leaf_threshold) + +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0,water_stock=0.0, carbon_stock=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) + +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if water levels are low, prioritise roots + if status.water_stock < m.water_leaf_threshold + return nothing + end + + # if not enough carbon, no organ creation + if status.carbon_stock < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end + +############################ +# Naive water absorption model +# Absorbs precipitation water depending on quantity of roots +############################ +PlantSimEngine.@process "water_absorption" verbose = false + +struct ToyWaterAbsorptionModel <: AbstractWater_AbsorptionModel +end + +PlantSimEngine.inputs_(::ToyWaterAbsorptionModel) = (root_water_assimilation=1.0,) +PlantSimEngine.outputs_(::ToyWaterAbsorptionModel) = (water_absorbed=0.0,) + +function PlantSimEngine.run!(m::ToyWaterAbsorptionModel, models, status, meteo, constants=nothing, extra=nothing) + #root_end = get_root_end_node(status.node) + #root_len = root_end[:Root_len] + status.water_absorbed = meteo.Precipitations * status.root_water_assimilation #* root_len +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsObjectIndependent() + + +########################## +### Root growth : when water stocks are low, expand root +########################## + +PlantSimEngine.@process "root_growth" verbose = false + +struct ToyRootGrowthModel{T} <: AbstractRoot_GrowthModel + water_threshold::T + carbon_root_creation_cost::T + root_max_len::Int +end + +PlantSimEngine.inputs_(::ToyRootGrowthModel) = (water_stock=0.0,carbon_stock=0.0,) +PlantSimEngine.outputs_(::ToyRootGrowthModel) = (carbon_root_creation_consumed=0.0,) + +function PlantSimEngine.run!(m::ToyRootGrowthModel, models, status, meteo, constants=nothing, extra=nothing) + if status.water_stock < m.water_threshold && status.carbon_stock > m.carbon_root_creation_cost + + root_end = get_root_end_node(status.node) + + if length(root_end) != 1 + throw(AssertionError("Couldn't find MTG leaf node with symbol \"Root\"")) + end + root_len = get_roots_count(root_end[1]) + if root_len < m.root_max_len + st = add_organ!(root_end[1], extra, "<", "Root", 2, index=1) + status.carbon_root_creation_consumed = m.carbon_root_creation_cost + end + else + status.carbon_root_creation_consumed = 0.0 + end +end + +########################## +### Model accumulating carbon and water resources +########################## + +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end +#status.water_stock += meteo.precipitations * root_water_assimilation_ratio + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(water_absorbed=0.0,carbon_captured=0.0,carbon_organ_creation_consumed=0.0,carbon_root_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (water_stock=-Inf,carbon_stock=-Inf) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.water_stock += sum(status.water_absorbed) #- status.water_transpiration + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) - sum(status.carbon_root_creation_consumed) + + if status.water_stock < 0.0 + status.water_stock = 0.0 + end +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsObjectIndependent() + +######################## +## Leaf model capturing some arbitrary carbon quantity +######################## + +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple()#(TT_cu=-Inf) +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + # very crude approximation with LAI of 1 and constant PPFD + status.carbon_captured = 200.0 *(1.0 - exp(-0.2)) +end + +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsTimeStepIndependent() + +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + :carbon_root_creation_consumed=>["Root"], + :carbon_organ_creation_consumed=>["Internode"] + + ], + ), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:water_stock)=>"Plant", + PreviousTimeStep(:carbon_stock)=>"Plant"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Root" => ( MultiScaleModel( + model=ToyRootGrowthModel(10.0, 50.0, 10), + mapped_variables=[PreviousTimeStep(:carbon_stock)=>"Plant", + PreviousTimeStep(:water_stock)=>"Plant"], + ), + ToyWaterAbsorptionModel(), + Status(carbon_root_creation_consumed=0.0, root_water_assimilation=1.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) + + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + plant_root_start = MultiScaleTreeGraph.Node( + plant, + MultiScaleTreeGraph.NodeMTG("+", "Root", 1, 3), + ) + + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +``` + +There are two major issues hinted at in last chapter's implementation, which we'll discuss and resolve here. + +You can find the full script for this simulation in the [ToyMultiScalePlantModel](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyMultiScalePlantModel/ToyPlantSimulation3.jl) subfolder of the examples folder. + +```@contents +Pages = ["multiscale_example_3.md"] +Depth = 3 +``` + +## An organ creation problem + +There is one quirk you may have noticed when inspecting the data : when a root expands, the new root is immediately active, and some models may act on it immediately... including the root growth model. Meaning this new root may also sprout another root in the same timestep, and so on. + +You can notice this by looking at the simulation's state after the first timestep: + +```@example usepkg +outs = run!(mtg, mapping, first(meteo_day, 2)) +nodes_per_timestep = outs["Root"][:node] +root_lengths_per_timestep = [length(nodes_per_timestep[i]) for i in 1:length(nodes_per_timestep)] + +``` + +Our root grew to full length within one timestep. Oops. + +This is an implementation decision in PlantSimEngine. **By default, newly created organs are active**, and models can affect them **as soon as they are created**. + +In our case, internode growth depends on a threshold thermal time value, which accumulates over several timesteps, so even though new internodes are immediately active, they can't themselves grow new organs within the same timestep. But as we've just showcased, we have a root problem. + +This quirk is also handled in [XPalm.jl](https://github.com/PalmStudio/XPalm.jl), a package using PlantSimEngine: some organs make use of state machines, and are considered "immature" when they are created. Immature organs cannot grow new organs until some conditions are met for their state to change. There are also other conditions governing organ emergence, such as specific threshold values relating to Thermal Time (see [here](https://github.com/PalmStudio/XPalm.jl/blob/433e1c47c743e7a53e764672818a43ed8feb10c6/src/plant/phytomer/leaves/phyllochron.jl#L46) for an example). + +!!! note + This implementation decision for new organs to be immediately active may be subject to change in future versions of PlantSimEngine. Also note that the way the dependency graph is structured determines the order in which models run. Meaning that which models are run before or after organ creation might change with new additions and updates to your mapping. Some models might run "one timestep later", see [Simulation order instability when adding models](@ref) for more details. + +!!! note + MTG node output data has a couple of subtleties, see [Multi-scale output data structure](@ref) for more details + +### Delaying organ maturity + +How do we avoid this extreme instant growth ? We can, of course, add some thermal time constraint. We could arbitrarily tinker with water resources. + +We can otherwise add a simple state machine variable to our root and internodes in the MTG, indicating a newly added organ is immature and cannot grow on the same timestep. Since our root doesn't branch, we can simply keep track of a single state variable. See the [State machines](@ref) section for some examples. + +In fact, we could change the scale at which the check is made to extend the root, and have another model call this one directly. This enables running this model only for the end root when those occasional timesteps when root growth is possible, instead of at every timestep for every root node. + +## A resource distribution bug + +Another problem you may have noticed, is that the water and carbon stock are computed by aggregating photosynthesis over leaves and absorption over roots... But they aren't always properly decremented when consumed ! + +If the end root grows, it outputs a `carbon_root_creation_consumed` value, but under certain conditions, we might also create other roots and internodes even when there shouldn't be enough carbon left for them. + +Indeed, if both the root and leaf water thresholds are met, and there is enough carbon for a single root or internode but not for both, and the root model runs before the internode model, both will use the carbon_stock variable prior to organ emission. The internode emission model won't account for the root carbon consumption. + +This occurs because `carbon_stock` is only computed once, and won't update until the next timestep. + +### Fixing resource computation: a root growth decision model + +To avoid that problem in our specific case, we can couple the root growth model and the internode emission model, and pass the `carbon_root_creation_consumed` variable to the internode emission model so that it can use an updated carbon stock. Or we could have an intermediate model recompute the new stock to pass along to the internode emission model. + +There is a section in the [Tips and workarounds] page discussing this situation and other potential solutions: [Having a variable simultaneously as input and output of a model](@ref). + +We'll go for the first option and couple the root growth and internode emission model. + +### Internode emission adjustments + +The only change required for our internode emission model is to take into account `carbon_root_creation_consumed` as a new input, map that variable from the "Root" scale in our mapping, and compute the adjusted carbon stock. Here's the relevant excerpt in the [`run!`](@ref) function. + +```julia + # take into account that the stock may already be depleted + carbon_stock_updated_after_roots = status.carbon_stock - status.carbon_root_creation_consumed + + # if not enough carbon, no organ creation + if carbon_stock_updated_after_roots < m.carbon_internode_creation_cost + return nothing + end +``` + +### A multi-scale hard dependency appears + +Our root growth decision model inherits some of the responsibility from last chapter's root growth model, so inputs, parameters and condition checks will be similar. We'll let the root growth model keep the length check and only focus on resources. + +Since the decision model is now directly responsible for calling the actual root growth model, we need to declare that it requires a root growth model as a hard dependency and cannot be run standalone. + +This hard dependency is in fact multiscale, since both models operate at different scales, "Plant" and "Root". You can read more about multi-scale hard dependencies in the [Handling dependencies in a multiscale context](@ref) page. + +Compared to the single-scale equivalent, the multi-scale declaration additionally requires mapping the scale: + +```julia +PlantSimEngine.dep(::ToyRootGrowthDecisionModel) = (root_growth=AbstractRoot_GrowthModel=>["Root"],) +``` + +The `status` argument [`run!`](@ref) function of the root growth decision model only contains variables from the "Plant" scale, or explicitely mapped to this scale, which isn't the case for the root growth's variables. To make use of the root growth model's variables, we need to recover the [`status`](@ref) at the "Root" scale. It is accessible from the `extra` argument in [`run!`](@ref)'s signature. + +In multi-scale simulations, this `extra` argument implicitely contains an object storing the simulation state. It contains the statuses at various scales, and all the models indexed per scale and process name. + +Access to the "Root" status within the root growth decision model [`run!`](@ref) function is done like so: + +```julia +status_Root= extra_args.statuses["Root"][1] +``` + +It is then possible to call the root growth model from the parent's [`run!`](@ref) function: + +```julia +PlantSimEngine.run!(extra.models["Root"].root_growth, models, status_Root, meteo, constants, extra) +``` + +Which will enable writing the rest of the [`run!`](@ref) function. + +### Root growth decision model implementation + +With that new coupling consideration properly handled, we can complete the full model implementation: + +```julia +PlantSimEngine.@process "root_growth_decision" verbose = false + +struct ToyRootGrowthDecisionModel{T} <: AbstractRoot_Growth_DecisionModel + water_threshold::T + carbon_root_creation_cost::T +end + +PlantSimEngine.inputs_(::ToyRootGrowthDecisionModel) = +(water_stock=0.0,carbon_stock=0.0) + +PlantSimEngine.outputs_(::ToyRootGrowthDecisionModel) = NamedTuple() + +PlantSimEngine.dep(::ToyRootGrowthDecisionModel) = (root_growth=AbstractRoot_GrowthModel=>["Root"],) + +# "status" is at the "Plant" scale +function PlantSimEngine.run!(m::ToyRootGrowthDecisionModel, models, status, meteo, constants=nothing, extra=nothing) + + if status.water_stock < m.water_threshold && status.carbon_stock > m.carbon_root_creation_cost + # Obtain "status" at "Root" scale + status_Root= extra_args.statuses["Root"][1] + # Call the hard dependency model directly with its status + PlantSimEngine.run!(extra.models["Root"].root_growth, models, status_Root, meteo, constants, extra) + end +end +``` + +The root growth model will output the `carbon_root_creation_consumed` computation, but it'll still be exposed to downstream models despite the root growth model being a 'hidden' model in the dependency graph due to its hard dependency nature. + +With this new coupling, we will only be creating at most a single new root per timestep, as the root growth decision will only be called once per timestep. + +### Root growth + +This iteration turns into a simplifed version of last chapter's. + +```julia +PlantSimEngine.@process "root_growth" verbose = false + +struct ToyRootGrowthModel <: AbstractRoot_GrowthModel + root_max_len::Int +end + +PlantSimEngine.inputs_(::ToyRootGrowthModel) = NamedTuple() +PlantSimEngine.outputs_(::ToyRootGrowthModel) = (carbon_root_creation_consumed=0.0,) + +function PlantSimEngine.run!(m::ToyRootGrowthModel, models, status, meteo, constants=nothing, extra=nothing) + status.carbon_root_creation_consumed = 0.0 + + root_end = get_root_end_node(status.node) + + if length(root_end) != 1 + throw(AssertionError("Couldn't find MTG leaf node with symbol \"Root\"")) + end + + root_len = get_roots_count(root_end[1]) + if root_len < m.root_max_len + st = add_organ!(root_end[1], extra, "<", "Root", 2, index=1) + status.carbon_root_creation_consumed = m.carbon_root_creation_cost + end +end +``` + +### Mapping adjustments + +The new mapping only has straightforward changes. Some models cease to be multi-scale, others require new variables to be mapped for them. `carbon_root_creation_consumed` ceases to be a vector mapping and is a scalar variable. + +```julia +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + :carbon_root_creation_consumed=>"Root", + :carbon_organ_creation_consumed=>["Internode"] + + ], + ), + MultiScaleModel( + model=ToyRootGrowthDecisionModel(10.0, 50.0), + ), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + :water_stock=>"Plant", + :carbon_stock=>"Plant", + :carbon_root_creation_consumed=>"Root"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Root" => (ToyRootGrowthModel(10), + ToyWaterAbsorptionModel(), + Status(carbon_root_creation_consumed=0.0, root_water_assimilation=1.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) +``` + +We can now run our simulation as we did previously... or can we ? + +```julia +ERROR: Cyclic dependency detected for process resource_stock_computation: resource_stock_computation for organ Plant depends on root_growth from organ Root, which depends on the first one. This is not allowed, you may need to develop a new process that does the whole computation by itself. +``` + +Ah, it looks like our additional usage of the root carbon cost creates a cyclic dependency. + +### Breaking the dependency cycle + +Fortunately, the logic here is quite straightforward. We can't be computing our current timestep's resource stock with `carbon_root_creation_consumed`, and then updating it right after root creation again using a new value of `carbon_root_creation_consumed`. + +The solution is hopefully quite intuitive : when we compute resource stocks, we should be computing it using the previous timestep's values. Then root creation happens (or doesn't), and the computed `carbon_root_creation_consumed` corresponds to the current timestep value. We could also do the same for water to be consistent. + +### Updated mapping + +The relevant part of the mapping that needs to be updated is the following: + +```julia +mapping = Dict( +... +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + PreviousTimeStep(:carbon_root_creation_consumed)=>"Root", + PreviousTimeStep(:carbon_organ_creation_consumed)=>["Internode"], + ], + ), + ToyRootGrowthDecisionModel(10.0, 50.0), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +... +) +``` + +## Final words + +And you're now ready to run the simulation. + +The full script can be found [here](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyMultiScalePlantModel/ToyPlantSimulation3.jl), in the ToyMultiScalePlantModel subfolder of the examples folder. + +We now have a plant with two different growth directions. Roots are added at the beginning, until water is considered abundant enough. + +Of course, there are still several design issues with this implementation. It is as utterly unrealistic as the previous one, and doesn't even consume water. Some condition checking is a little ad hoc and could be made more robust. More sanity checks could be added, and the model and variable names could definitely be made more clear. + +But once again, this example is only made to illustrate what is possible with this framework, and doesn't strive for ecophysiological consistency. And the approach can be made increasingly more complex by refining models and simulation parameters, and feeding in new information about your plant, and ramp up to realistic, production-ready and predictive simulations. \ No newline at end of file diff --git a/docs/src/multiscale/multiscale_example_4.md b/docs/src/multiscale/multiscale_example_4.md new file mode 100644 index 000000000..cdf8c5889 --- /dev/null +++ b/docs/src/multiscale/multiscale_example_4.md @@ -0,0 +1,226 @@ +# Visualizing a plant using PlantGeom + +We've created our toy plant, part of the fun is to actually visualize it ! + +Let's see how to do so with the [PlantGeom](https://github.com/VEZY/PlantGeom.jl) companion package. + +We'll be reusing the mtg from part 3 of the plant tutorial: [Fixing bugs in the plant simulation](@ref), so you need to run that simulation first, or to include the script file into your current code (which is what we'll do here): + +```julia +using PlantSimEngine +using MultiScaleTreeGraph +using PlantSimEngine.Examples +using Pkg +Pkg.add("CSV") +using CSV +include("ToyPlantSimulation3.jl") +``` + +You'll need to add PlantGeom and a compatible visualization package to your environment. We'll use Plots: + +```julia +using Plots +using PlantGeom +``` + +That's enough to get a nicer display of the MTG than the console-based printing. You'll only need to type the following line: + +```julia +RecipesBase.plot(mtg) +``` + +This provides the following visualization: +![MTG Plots visualization](../www/mtg_plot_1.svg) + +And that's it ! + +We can see the root expansion in one direction, and the internodes with their leaves in the other. + +Of course, that's good and all, but going beyond that would be nice. + +PlantGeom is able to render geometry from what it finds in the MTG. If a node in the tree graph has a `:geometry` attribute with a mesh and a transformation, it can make use of that to build a plant. That mesh can be unique per node, or based on a reference mesh that is copied and transformed for every node. + +!!! note + This page simply aims to illustrate PlantGeom's features and doesn't aim for a particularly realistic or aesthetic look. A little randomness could go a long way to make the plant look a little more life-like, but would also be very ad-hoc and make the code less clear. + +Our MTG doesn't have any such attribute, so we'll need to iterate on our nodes, provide them with a mesh and calculate appropriate transformations. We'll use one reference mesh for each plant-related scale, Internode, Root and Leaf. + +We'll make use of some of the Meshes primitives and transformations, as well as some helper functions from the packages TransformsBase and Rotations. For leaves, we'll read a .ply file using the PlyIO package, which contains a very unrealistic leaf + petiole mesh. + +We'll also make our plant opposite decussate: leaves come in pairs, and pairs are rotated by 90 degrees along the stem. + +The function that'll provide the geometry to the node is: +```julia +PlantGeom.Geometry(; ref_mesh<:RefMesh, transformation=Identity(), dUp=1.0, dDwn=1.0, mesh::Union{SimpleMesh,Nothing}=nothing) +``` + +We only care about the first two parameters in our case, and we can use a simple cylinder for each node of our single Internode stem and single Root. + +```julia +using PlantGeom.Meshes + +# Internodes and roots will use a cylinder as a mesh + +cylinder() = Meshes.CylinderSurface(1.0) |> Meshes.discretize |> Meshes.simplexify + +refmesh_internode = PlantGeom.RefMesh("Internode", cylinder()) +refmesh_root = PlantGeom.RefMesh("Root", cylinder()) +``` + +A simple function to read the vertices and faces from the .ply file for our leaves: + +```julia +Pkg.add("PlyIO") +using PlyIO +function read_ply(fname) + ply = PlyIO.load_ply(fname) + x = ply["vertex"]["x"] + y = ply["vertex"]["y"] + z = ply["vertex"]["z"] + points = Meshes.Point.(x, y, z) + connec = [Meshes.connect(Tuple(c .+ 1)) for c in ply["face"]["vertex_indices"]] + Meshes.SimpleMesh(points, connec) +end + +leaf_ply = read_ply("examples/leaf_with_petiole.ply") +refmesh_leaf = PlantGeom.RefMesh("Leaf", leaf_ply) +``` + +```julia +Pkg.add("TransformsBase") +Pkg.add("Rotations") +import TransformsBase: → +import Rotations: RotY, RotZ, RotX +``` + +!!! note + We'll use X, Y, Z as standard cartersian coordinate axes, with Z pointing upwards. + +We can then write the function that adds the geometry to our MTG. + +It traverses the MTG, starting from the base, and adds a transformation for each encountered node. + +The following just operates on internodes, for clarity: + +```julia +# Add the geometry to the MTG, with transformations +function add_geometry!(mtg, refmesh_internode) + + # incremental offset + internode_height = 0.0 + + # relative scale of the base mesh + internode_width = 0.5 + + # length of the base mesh + internode_length = 1.0 + + traverse!(mtg) do node + if symbol(node) == "Internode" + # Set to scale, then translate by the total height + mesh_transformation = Meshes.Scale(internode_width, internode_width, internode_length) → Meshes.Translate(0.0, 0.0, internode_height) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_internode, transformation=mesh_transformation) + + internode_height += node_length + end + end +end +``` + +We simply need to choose a given width for our stem, and increment the height to place our next internode at as we traverse it. + +Note that the default cylinder provided by Meshes.jl points upwards, which is why there is no need for rotation. Roots function likewise, but are simply translated down, and need to start below the origin. + +We can visualize this simple stem, using GLMakie as a rendering backend: + +```julia +add_geometry!(mtg, refmesh_internode) + +# Visualize the mesh +using GLMakie +viz(mtg) +``` + +![Toy Plant - stem only](../www/toy_plant_stem_only.png) + +On the other hand, the leaf mesh will need to be rotated, but it is aligned along the X axis, so there is no need for an initial reorientation (which would have been required if it was pointing upwards like the cylinders). The petiole starts at the origin, so on top of translating them to leaf height we also need to translate them away from the Z axis by the internode radius. The mesh also needs to be scaled, as it is only 0.1 unit lengths long compared to our 0.5-width internode. + +Let's also rotate our leaves so that they point upwards slightly. + +If you make use of other meshes, bear in mind the initial starting translation, orientation and scale. You may need to test and calibrate scales and transformations before you get it right. + +The full code that generates geometry for all the organs of our toy plant is the following: +```julia +# Add the geometry to the MTG, with transformations +function add_geometry!(mtg, refmesh_internode, refmesh_root, refmesh_leaf) + + # incremental offset + internode_height = 0.0 + root_depth = 0.0 + + # relative scale of the base mesh (base cylinder is of height 1 and radius 1) + internode_width = 0.5 + root_width = 0.2 + + # length of the base mesh + internode_length = 1.0 + root_length = 1.0 + + # ad hoc value to adjust the leaf mesh to the scene scale + leaf_mesh_scale = 25 + + leaf_scale_width = 0.4*leaf_mesh_scale + leaf_scale_height = 0.4*leaf_mesh_scale + + # Helpers to make the leaves opposite decussate + leaf_rotation = MathConstants.pi / 2.0 + i = 0 + + traverse!(mtg) do node + if symbol(node) == "Internode" + # Set to scale, then translate by the total height + mesh_transformation = Meshes.Scale(internode_width, internode_width, internode_length) → Meshes.Translate(0.0, 0.0, internode_height) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_internode, transformation=mesh_transformation) + + internode_height += internode_length + + # Leaves are placed relatively to the parent internode, halfway along it + for chnode in children(node) + if symbol(chnode) == "Leaf" + mesh_transformation = Meshes.Scale(leaf_scale_width, leaf_scale_width, leaf_scale_height) → Meshes.Rotate(RotX(-MathConstants.pi / 6.0)) → Meshes.Translate(0.0, -internode_width, internode_height - internode_length / 2.0) → Meshes.Rotate(RotZ(leaf_rotation)) + chnode.geometry = PlantGeom.Geometry(ref_mesh=refmesh_leaf, transformation=mesh_transformation) + # Set the second leaf in a pair opposite to the first one => add a 180° rotation + leaf_rotation += MathConstants.pi + end + end + + # Opposite decussate => 90° rotation between pairs + i += 1 + if i % 2 == 0 + leaf_rotation = MathConstants.pi / 2.0 + else + leaf_rotation = MathConstants.pi + end + + elseif symbol(node) == "Root" + mesh_transformation = Meshes.Scale(root_width, root_width, root_length) → Meshes.Translate(0.0, 0.0, root_depth) → Meshes.Rotate(RotZ(MathConstants.pi)) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_root, transformation=mesh_transformation) + root_depth -= root_length + end + end +end +``` + +And now, let's visualize our fully-grown, fully-featured plant: + +```julia +# Visualize the mesh +using GLMakie +viz(mtg) +``` + +Which gives us the following image scene: + +![Toy Plant with root and leaves](../www/toy_plant.png) + +Feel free to try and make this plant prettier, more colourful, or more physically realistic, using more realistic models on the PlantSimEngine side, or better geometry on the Plantgeom end. \ No newline at end of file diff --git a/docs/src/multiscale/single_to_multiscale.md b/docs/src/multiscale/single_to_multiscale.md new file mode 100644 index 000000000..169e8260c --- /dev/null +++ b/docs/src/multiscale/single_to_multiscale.md @@ -0,0 +1,238 @@ +# Converting a single-scale simulation to multi-scale +```@meta +CurrentModule = PlantSimEngine +``` +```@setup usepkg +using PlantMeteo +using PlantSimEngine +using PlantSimEngine.Examples +using CSV +using DataFrames +using MultiScaleTreeGraph +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +models_singlescale = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) +``` + +A single-scale simulation can be turned into a 'pseudo-multi-scale' simulation by providing a simple multi-scale tree graph, and declaring a mapping linking all models to a unique scale level. + +This page showcases how to do the conversion, and then adds a model at a new scale to make the simulation genuinely multi-scale. + +The full script for the example can be found in the examples folder, [here](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToySingleToMultiScale.jl) + +```@contents +Pages = ["single_to_multiscale.md"] +Depth = 3 +``` + +# Converting the ModelList to a multi-scale mapping + +For example, let's return to the [`ModelList`](@ref) coupling a light interception model, a Leaf Area Index model, and a carbon biomass increment model that was discussed in the [Model switching](@ref) subsection: + +```@example usepkg +using PlantMeteo +using PlantSimEngine +using PlantSimEngine.Examples +using CSV + +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models_singlescale = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +outputs_singlescale = run!(models_singlescale, meteo_day) +``` + +Those models all operate on a simplified model of a single plant, without any organ-local information. We can therefore consider them to be working at the 'whole plant' scale. Their variables also operate at that "plant" scale, so there is no need to map any variable to other scales. + +We can therefore convert this into the following mapping : + +```@example usepkg +mapping = Dict( +"Plant" => ( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + Status(TT_cu=cumsum(meteo_day.TT),) + ), +) +``` +Note the slight difference in syntax for the [`Status`](@ref). This is due to an implementation quirk (sorry). + +## Adding a new package for our plant graph + +None of these models operate on a multi-scale tree graph, either. There is no concept of organ creation or growth. We still need to provide a multi-scale tree graph to a multi-scale simulation, so we can -for now- declare a very simple MTG, with a single node: + +```@example usepkg +using MultiScaleTreeGraph + +mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 0, 0),) +``` + +!!! note + You will need to add the `MultiScaleTreeGraph` package to your environment. See [Installing and running PlantSimEngine](@ref) if you are not yet comfortable with Julia or need a refresher. + +## Running the multi-scale simulation ? + +We now have **almost** everything we need to run the multiscale simulation. + +This first conversion step can be a starting point for a more elaborate multi-scale simulation. + +The signature of the [`run!`](@ref) function in multi-scale differs slightly from the ModelList version : + +```julia +out_multiscale = run!(mtg, mapping, meteo_day) +``` + +(Some of the optional arguments also change slightly) + +Unfortunately, there is one caveat. Passing in a vector through the [`Status`](@ref) field is still possible in multi-scale mode, but requires a little more advanced tinkering with the mapping, as it generates a custom model under the hood and the implementation is experimental and less user-friendly. + +If you are keen on going down that path, you can find a detailed example [here](@ref multiscale_vector), but we don't recommend it for beginners. + +What we'll do instead, is write our own model provide the thermal time per timestep as a variable, instead of as a single vector in the [`Status`](@ref). + +Our 'pseudo-multiscale' first approach will therefore turn into a genuine multi-scale simulation. + +## Adding a second scale + +Let's have a model provide the Cumulated Thermal Time to our Leaf Area Index model, instead of initializing it through the [`Status`](@ref). + +Let's instead implement our own `ToyTT_cuModel`. + +### TT_cu model implementation + +This model doesn't require any outside data or input variables, it only operates on the weather data and outputs our desired TT_cu. The implementation doesn't require any advanced coupling and is very straightforward. + +```@example usepkg +PlantSimEngine.@process "tt_cu" verbose = false + +struct ToyTt_CuModel <: AbstractTt_CuModel +end + +function PlantSimEngine.run!(::ToyTt_CuModel, models, status, meteo, constants, extra=nothing) + status.TT_cu += + meteo.TT +end + +function PlantSimEngine.inputs_(::ToyTt_CuModel) + NamedTuple() # No input variables +end + +function PlantSimEngine.outputs_(::ToyTt_CuModel) + (TT_cu=0.0,) +end +``` + +!!! note + The only accessible variables in the [`run!`](@ref) function via the status are the ones that are local to the "Scene" scale. This isn't explicit at first glance, but very important to keep in mind when developing models, or using them at different scales. If variables from other scales are required, then they need to be mapped via a [`MultiScaleModel`](@ref), or sometimes a more complex coupling is necessary. + +### Linking the new TT_cu model to a scale in the mapping + +We now have our model implementation. How does it fit into our mapping ? + +Our new model doesn't really relate to a specific organ of our plant. In fact, this model doesn't represent a physiological process of the plant, but rather an environmental process affecting its physiology. We could therefore have it operate at a different scale unrelated to the plant, which we'll call "Scene". This makes sense. + +Note that we now need to add a "Scene" node to our Multi-scale Tree Graph, otherwise our model will not run, since no other model calls it and "Plant" nodes will only call models at the "Plant" scale. See [Empty status vectors in multi-scale simulations](@ref) for more details. + +```@example usepkg +mtg_multiscale = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 0, 0),) + plant = MultiScaleTreeGraph.Node(mtg_multiscale, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) +``` + +### Mapping between scales : the MultiScaleModel wrapper + +The cumulated thermal time (`:TT_cu`) which was previously provided to the LAI model as a simulation parameter now needs to be mapped from the "Scene" scale level. + +This is done by wrapping our ToyLAIModel in a dedicated structure called a [`MultiScaleModel`](@ref). A [`MultiScaleModel`](@ref) requires two keyword arguments : `model`, indicating the model for which some variables are mapped, and `mapped_variables`, indicating which scale link to which variables, and potentially renaming them. + +There can be different kinds of variable mapping with slightly different syntax, but in our case, only a single scalar value of the TT_cu is passed from the "Scene" to the "Plant" scale. + +This gives us the following declaration with the [`MultiScaleModel`](@ref) wrapper for our LAI model: + +```@example usepkg +MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ) +``` +and the new mapping with two scales: + +```@example usepkg +mapping_multiscale = Dict( + "Scene" => ToyTt_CuModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.5), + ToyRUEGrowthModel(0.2), + ), +) +``` + +### Running the multi-scale simulation + +We can then run the multiscale simulation, with our two-node MTG : + +```@example usepkg +outputs_multiscale = run!(mtg_multiscale, mapping_multiscale, meteo_day) +``` + +### Comparing outputs between single- and multi-scale + +The outputs structures are slightly different : multi-scale outputs are indexed by scale, and a variable has a value for every node of the scale it operates at (for instance, there would be a "leaf_surface" value for every leaf in a plant), stored in an array. + +In our simple example, we only have one MTG scene node and one plant node, so the arrays for each variable in the multi-scale output only contain one value. + +We can access the output variables at the "Scene" scale by indexing our outputs: + +```@example usepkg +outputs_multiscale["Scene"] +``` +and then the computed `:TT_cu`: +```@example usepkg +outputs_multiscale["Scene"][:TT_cu] +``` + +As you can see, it is a `Vector{Vector{T}}`, whereas our single-scale output is a `Vector{T}`: +```@example usepkg +outputs_singlescale.TT_cu +``` + +To compare them value-by-value, we can flatten the multiscale Vector and then do a piecewise approximate equality test : +```@example usepkg +computed_TT_cu_multiscale = collect(Base.Iterators.flatten(outputs_multiscale["Scene"][:TT_cu])) + +for i in 1:length(computed_TT_cu_multiscale) + if !(computed_TT_cu_multiscale[i] ≈ outputs_singlescale.TT_cu[i]) + println(i) + end +end +``` +or equivalently, with broadcasting, we can write : +```@example usepkg +is_approx_equal = length(unique(computed_TT_cu_multiscale .≈ outputs_singlescale.TT_cu)) == 1 +``` + +!!! note + You may be wondering why we check for approximate equality rather than strict equality. The reason for that is due to floating-point accumulation errors, which are discussed in more detail in [Floating-point considerations](@ref). + +## ToyDegreeDaysCumulModel + +There is a model able to provide Thermal Time based on weather temperature data, [`ToyDegreeDaysCumulModel`](@ref), which can also be found in the examples folder. + +We didn't make use of it here for learning purposes. It also computes a thermal time based on default parameters that don't correspond to the thermal time in the example weather data, so results differ from the thermal time already present in the weather data without tinkering with the parameters. \ No newline at end of file diff --git a/docs/src/planned_features.md b/docs/src/planned_features.md new file mode 100644 index 000000000..cf2d8a90f --- /dev/null +++ b/docs/src/planned_features.md @@ -0,0 +1,55 @@ +# Roadmap + +## Planned major features + +### Varying timesteps + +Currently, all models are required to make use of the same timestep. Some physiological phenomenae within a plant tend to run on an hourly basis, others are slower. Weather data is often provided daily. Enabling different timesteps depending on the model is on the roadmap, and is planned as the next milestone. + +### Multi-plant/Multi-species simulations + +A goal for PlantSimEngine down the line is to be able to simulate complex scenes with data comprising several plants, possibly of different species, for agroforestry purposes. + +Its current state doesn't enable practical declaration of several plant species, or multiple plants relying on similar subsets of models with partially different models or parameters. + +## Minor features + +- Implement a trait or a prepass that checks whether weather data is needed, and if so, if it is properly provided to a simulation +- Better dependency graph visualization and information printing + +## Minor planned improvements and QOL features + +- A reworked and more consistent mapping API, and multiscale dependency declaration +- Improved user errors +- More examples +- Better dependency graph traversal functions +- Ensure cyclic dependency checking and PreviousTimestep is active for ModelLists + +## Improvements on the testing side + +- Better tracking of memory usage and type stability +- Working CI/Downstream tests +- state machine checker, validating output invariants +- graph fuzzing for improved corner-case testing + +## Possible features (likely not a priority) + +- API enabling iterative builds and validation of mappings and ModelLists +- Build step for the models, *i.e.* a function that would write a mapping or ModelList into a Julia script for validation, improved readability and (maybe) performance (no need to traverse the dependency graph anymore). +- Improved parallelisation +- Reintroduce multi-object parallelisation in single-scale + +## Other minor points + +- Examples/solutions for floating-point accumulation errors +- More examples for fitting/type conversion/error propagation +- MTG couple of new features #106 +- Other minor bugs +- Unrolling the run! function + +## Other + +- Reproducing another FSPM? +- Diffusion model example? + +The full list of issues can be found [here](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues) \ No newline at end of file diff --git a/docs/src/prerequisites/installing_plantsimengine.md b/docs/src/prerequisites/installing_plantsimengine.md new file mode 100644 index 000000000..2c8f85bf8 --- /dev/null +++ b/docs/src/prerequisites/installing_plantsimengine.md @@ -0,0 +1,78 @@ +# Installing and running PlantSimEngine + +```@contents +Pages = ["installing_plantsimengine.md"] +Depth = 3 +``` + +This page is meant to help along people newer to Julia. If you are quite accustomed to Julia, installing PlantSimEngine should be par for the course, and you can [move on to the next section](#step_by_step), or read about PlantSimEngine's [Key Concepts](@ref). + +## Installing Julia + +The direct download link can be found [here](https://julialang.org/downloads/), and some additional pointers [in the official manual](https://docs.julialang.org/en/v1/manual/installation/). + +## Installing VSCode + +You can get by using a REPL, but if writing a larger piece of software you may prefer using an IDE. PlantSimEngine is developed using VSCode, which you can install by following instruction [on this page](https://code.visualstudio.com/docs/setup/setup-overview). A documentation section specific to using Julia in VSCode can be found [here](https://code.visualstudio.com/docs/languages/julia). + +## Installing PlantSimEngine and its dependencies + +### Julia environments + +Julia package management is done via the Pkg.jl package. You can find more in-depth sections detailing its usage, and working with Julia environments [in its documentation](https://pkgdocs.julialang.org/v1/) + +If you find this page insufficient to get started, [this tutorial](https://jkrumbiegel.com/pages/2022-08-26-pkg-introduction/) explains in detail the subtleties of Julia environments. + +### Running an environment + +Once your environment is set up, you can launch a command prompt and type `julia`. This will launch Julia, and you should see `julia>` in the command prompt. + +You can always type `?` from there to enter help mode, and type the name of a function or language feature you wish to know more about. + +You can find out which directory you are in by typing `pwd()` in a Julia session. + +Handling environments and dependencies is done in Julia through a specific Package called Pkg, which comes with the base install. You can either call Pkg features the same way you would for another package, or enter Pkg mode by typing `]`, which will change the display from `julia>` to something like `(@v1.11)` pkg>, indicating your current environment (in this case, the default julia environment, which we don't recommend bloating). + +Once in Pkg mode, you can choose to create an environment by typing `activate path/to/environment`. + +You can then add packages that have been added to Julia's online global registry by typing `add packagename` and you can remove them by typing `remove packagename`. Typing `status` or `st` will indicate what your current environment is comprised of. To update packages in need of updating (a `^` symbol will display next to their name), type `update`… or `up`. + +If you are editing/developing a package or using one locally, typing `develop path/to/package source/` (or `dev path/to/package/source`) will cause your environment to use that version instead of the registered one. + +Typing `instantiate` will download all the packages declared in the manifest file (if it exists) of an environment. + +For instance, PlantSimEngine has a test folder used in development. If you wanted to run tests, you would type `]` then `activate ../path/to/PlantSimEngine/test` then `instantiate` +and then you would be ready to run some scripts. + +So if you wish to use PlantSimEngine, you can enter Pkg mode (`]`), choose an environment folder, then activate that environment with `activate ../path/to/your_environment`, add PlantSimEngine to it with `add PlantSimEngine` then download the package and its dependencies with `instantiate`. + +### Companion packages + +You'll also, for most of our examples, need `PlantMeteo`. For several multi-scale simulations, you'll need `MultiScaleTreeGraph`. + +Some of the weather data examples make use of the `CSV` package, some output data is manipulated as a DataFrame, which is part of the `DataFrames` package. + +### Using the example models + +Example models are exported as a distinct submodule of PlantSimEngine, meaning they aren't part of the main API. You can use them by typing: + +```julia +using PlantSimEngine.Examples +``` + +## Running a test simulation + +Assuming you've setup you're environement, correctly added `PlantMeteo` and `PlantSimEngine` to that environment, and downloaded everything with `instantiate`, you'll be able to run a test example in your REPL by typing line-by-line: + +```@example mypkg +using PlantSimEngine, PlantMeteo +using PlantSimEngine.Examples +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) +leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) +out_sim = run!(leaf, meteo) +``` + +## Environments in VSCode + +There is detailed documentation explaining how to make use of Julia with VSCode with one section indicating how to handle environments in VSCode: [https://www.julia-vscode.org/docs/stable/userguide/env/](https://www.julia-vscode.org/docs/stable/userguide/env/) + \ No newline at end of file diff --git a/docs/src/prerequisites/julia_basics.md b/docs/src/prerequisites/julia_basics.md new file mode 100644 index 000000000..bb22f92ee --- /dev/null +++ b/docs/src/prerequisites/julia_basics.md @@ -0,0 +1,55 @@ +# Getting started with Julia + +PlantSimEngine (as well as its related packages) is written in Julia. The reasons why Julia was chosen are briefly discussed here : [The choice of using Julia](@ref). + +Julia is a language that is gaining traction, but it isn't the most widely used in research and data science. + +Many elements will be familiar to those with an R, Python or Matlab background, but there are some noteworthy differences, and if you are new to the language, there will be a few hurdles you might have to overcome to be comfortable using the language. + +This page is here to list to the parts of Julia that are most relevant regarding usage of PlantSimEngine, and point to resources that can help you grasp those basics. + +## New to programming + +It is not meant as a full-fledged from-scratch Julia tutorial. If you are completely new to programming, you may wish to check some other resources first, such as ones found [here](https://docs.julialang.org/en/v1/manual/getting-started/). The video course [Julia Programming for Nervous Beginners](https://www.youtube.com/playlist?list=PLP8iPy9hna6Qpx0MgGyElJ5qFlaIXYf1R) is tailored for people with no programming experience. + +## Installing packages and setting up and environment + +For PlantSimEngine, you can check our documentation page on the topic: +[Installing and running PlantSimEngine](@ref) + +## Cheatsheets + +You can also find a few cheatsheets [here](https://palmstudio.github.io/Biophysics_database_palm/cheatsheets/) as well as a [short introductory notebook](https://palmstudio.github.io/Biophysics_database_palm/basic_syntax/) along with its [install instructions](https://palmstudio.github.io/Biophysics_database_palm/installation/). + +## Troubleshooting + +There is a documentation page showcasing some of the common errors than can occur when using PlantSimEngine, which may be worth checking if you are encountering issues: [Troubleshooting error messages](@ref). + +For more Julia learning-related difficulties, you will find quick responses on the Discourse forum: [https://discourse.julialang.org](https://discourse.julialang.org). + +### Noteworthy differences with other languages: + +If you wish to compare Julia to a specific language, [the noteworthy differences section](https://docs.julialang.org/en/v1/manual/noteworthy-differences/#Noteworthy-differences-from-Python) will provide you with a quick overview of the differences. + +(Array indexing starts at 1, for example) + +## Essential Julia concepts for PlantSimEngine + +Here's a list of the main aspects of the Julia language required (beyond package management) to understand how to use PlantSimEngine to its potential: + +Standard notions and constructs: + +- Standard concepts of a variable, arrays, functions, function arguments +- The typing system and custom types +- Dictionaries and NamedTuple objects are used throughout the codebase + +The Julia manual goes more in-depth than lighter introductions to some of these topics, so might be more useful as a reference than a starting point. You might find other guides or courses, such as the first section in [https://julia.quantecon.org/intro.html](https://julia.quantecon.org/intro.html), chapters 0-4 and 7 of the [Learn Julia the Hard Way](https://scls.gitbooks.io/ljthw/content/) draft or the interactive [Mathigon course](https://mathigon.org/course/programming-in-julia/introduction). + +Also of importance: + +- [Keyword arguments](https://docs.julialang.org/en/v1/manual/functions/#Keyword-Arguments) (kwargs) are present in many API functions +- [Type promotion](https://docs.julialang.org/en/v1/manual/conversion-and-promotion/#Promotion), [splatting](https://docs.julialang.org/en/v1/base/base/#...), [broadcasting](https://docs.julialang.org/en/v1/manual/functions/#man-vectorized), and [comprehensions](https://docs.julialang.org/en/v1/manual/arrays/#man-comprehensions) are also very useful, but not compulsory to get started + +Many of these are also briefly presented in [this Julia Data Science](https://juliadatascience.io/julia_basics) guide, which also happens to focus on the DataFrames.jl package. + +Understanding more about methods, parametric types and the typing system is usually worthwhile, when working with Julia packages. \ No newline at end of file diff --git a/docs/src/prerequisites/key_concepts.md b/docs/src/prerequisites/key_concepts.md new file mode 100644 index 000000000..9021062fa --- /dev/null +++ b/docs/src/prerequisites/key_concepts.md @@ -0,0 +1,189 @@ +# Key Concepts + +You'll find a brief description of some of the main concepts and terminology related to and used in PlantSimEngine. + +```@contents +Pages = ["key_concepts.md"] +Depth = 4 +``` + +## Crop models + +## FSPM + +## PlantSimEngine terminology + +This page provides a general description of the concepts and terminology used in PlantSimEngine. For a more implementation-guided description of the design and some of the terms presented here, see the [Detailed walkthrough of a simple simulation](@ref detailed-walkthrough-of-a-simple-simulation) + +!!! Note + Some terminology has different meanings in different contexts. This is particularly true of the terms organ, scale and symbol, which have a different meaning for [Multi-scale Tree Graphs](@ref) than the rest of PlantSimEngine (see [Scale/symbol terminology ambiguity](@ref) further down). Make sure to double-check those subsections, and relevant examples if you encounter issues relating to these terms. + +### Processes + +A process in this package defines a biological or physical phenomena. Think of any process happening in a system, such as light interception, photosynthesis, water, carbon and energy fluxes, growth, yield or even electricity produced by solar panels. + +See [Implementing a new process](@ref) for a brief explanation on how to declare a new process. + +### Models + +A model is a particular implementation for the simulation of a process. + +There may be different models that can be used for the same process; for instance, there are multiple hypotheses and ways of modeling photosynthesis, with different granularity and accuracy. A simple photosynthesis model might apply a simple formula and apply it to the total leaf surface, a more complex one might calculate interception and light extinction. + +!!! note + The companion package PlantBiophysics.jl provides the [`Beer`](https://vezy.github.io/PlantBiophysics.jl/stable/functions/#PlantBiophysics.Beer) structure for the implementation of the Beer-Lambert law of light extinction. The process of `light_interception` and the `Beer` model are provided as an example script in this package too at [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/Beer.jl). + +Models can also be used for ad hoc computations that aren't directly tied to a specific literature-defined physiological process. In PlantSimEngine, everything is a model. There are many instances where a custom model might be practical to aggregate some computations or handle other information. To illustrate, XPalm, the Oil Palm model, has a few models that handle the state of different organs, and a model to handle leaf pruning, which you can find [here](https://github.com/PalmStudio/XPalm.jl/blob/main/src/plant/phytomer/leaves/leaf_pruning.jl). + +To prepare a simulation, you declare a ModelList with whatever models you wish to make use of and initialize necessary parameters: see the [step by step](@ref detailed-walkthrough-of-a-simple-simulation) section to learn how to use them in practice. + +For multi-scale simulations, models need to be tied to a particular scale when used. See the [Multiscale modeling](@ref) section below, or the [Multi-scale considerations](@ref) page for a more detailed description of multi-scale peculiarities. + +### Variables, inputs, outputs, and model coupling + +A model used in a simulation requires some input data and parameters, and will compute some other data which may be used by other models. Depending on what models are combined in a simulation, some variables may be inputs of some models, outputs of other models, only be part of intermediary computations, or be a user input to the whole simulation. + +Here's a conceptual model coupling; each "node" is equivalent to a distinct PlantSimEngine model, "compute()" is equivalent to the model's "run!" function: + +![Model coupling example](../www/GUID-12E2DDAD-7B20-4FE2-AA36-7FAC950382A6-low.png) +(Source: [Autodesk](https://help.autodesk.com/view/MAYAUL/2016/ENU/?guid=__files_GUID_A9070270_9B5D_4511_8012_BC948149884D_htm")) + +### Dependency graphs + +Coupling models together in this fashion creates what is known as a [Directed Acyclic Graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) or DAG, a type of [dependency graph](https://en.wikipedia.org/wiki/Dependency_graph). The order in which models are run is determined by the ordering of these models in that graph. + +![Example DAG](../www/dags_acyclic_vs_cyclic-d1a669bf1b8b6bfa8ac3041788e81171.png) +A simple Directed Acyclic Graph, note the required absence of cycles. Source: [Astronomer](https://www.astronomer.io/docs/learn/dags/) (Note: "Not Acyclic" is simply "Cyclic"). + +PlantSimEngine creates this Directed Acyclic Graph automatically by plugging the right variables in the right models. Users therefore only need to declare models, they do not need to write the code to connect them as PlantSimEngine does that work for them, as long as the model coupling has no cyclic dependency. + +### ["Hard" and "Soft" dependencies](@id hard_dependency_def) + +Linking models by setting output variables from one model as input of another model handles many typical couplings (with more situations occurring with multi-scale models and variables), but what if two models are interdependent? What if they need to iterate on some computation and pass variables back and forth? + +You can find a typical example in a companion package: [PlantBioPhysics.jl](https://github.com/VEZY/PlantBiophysics.jl). An energy balance model, the [Monteith model](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl), needs to [iteratively run a photosynthesis model](https://github.com/VEZY/PlantBiophysics.jl/blob/c1a75f294109d52dc619f764ce51c6ca1ea897e8/src/processes/energy/Monteith.jl#L154) in its [`run!`](@ref) function. + +See the illustration below of the way these models are interdependent: + +![Example of a coupling with cycles](../www/ecophysio_coupling_diagram.png) + +Example of a coupling with a cycle. Source: PlantBioPhysics.jl + +Model couplings that cause simulation to flow both ways break the 'acyclic' assumption of the dependency graph. + +PlantSimEngine handles this internally by not having those "heavily-coupled" models -called **hard dependencies** from now on- be part of the main dependency graph. Instead, modelers should call these models manually from within a model. This way, they are made to be children nodes of the parent/ancestor model, which handles them internally, so they aren't tied to other nodes of the dependency graph. The resulting higher-level graph therefore only links models without any two-way interdependencies, and remains a directed graph, enabling a cohesive simulation order. The simpler couplings in that top-level graph are called "soft dependencies". + +![Hard dependency coupling visualization in PlantSimEngine](../www/PBP_dependency_graph.png) +The previous coupling, handled by PlantSimEngine + +How PlantSimEngine links these models under the hood. The red models ("hard dependencies") are not exposed in the final dependency graph, which only contains the blue "soft dependencies", and has no cycles. + +This approach does have implications when developing interdependent models: hard dependencies need to be made explicit, and the ancestor needs to call the hard dependency model's [`run!`](@ref) function explicitely in its own [`run!`](@ref) function. Hard dependency models therefore must have only one parent model. + +This reliance on another process makes these models slightly more complex to develop and validate, but keep the versatility of the implementation, as any model implementing the hard-dependency process can be passed by the user. + +Note that hard dependencies can also have their own hard dependencies, and some complex couplings can happen. + +### Weather data + +To run a simulation, we usually need the climatic/meteorological conditions measured close to the object or component. + +Users are strongly encouraged to use [`PlantMeteo.jl`](https://github.com/PalmStudio/PlantMeteo.jl), the companion package that helps manage such data, with default pre-computations and structures for efficient computations. We will make constant use of it throughout the documentation, and recommend working with it. + +The most basic data structure from this package is a type called [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere), which defines steady-state atmospheric conditions, *i.e.* the conditions are considered at equilibrium. Another structure is available to define different consecutive time-steps: [`TimeStepTable`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.TimeStepTable). + +The mandatory variables to provide for an [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere) are: `T` (air temperature in °C), `Rh` (relative humidity, 0-1) and `Wind` (the wind speed, m s⁻¹). + +In the example below, we also pass in the -optional- incoming photosynthetically active radiation flux (`Ri_PAR_f`, W m⁻²). We can declare such conditions like so: + +```@example usepkg +using PlantMeteo +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) +``` + +More details are available from the [package documentation](https://vezy.github.io/PlantMeteo.jl/stable). If you do not wish to make use of this package, you can alternately provide your own data, as long as it respects the [Tables.jl interface](https://tables.juliadata.org/stable/#Implementing-the-Interface-(i.e.-becoming-a-Tables.jl-source)) (*e.g.* use a `DataFrame`). + +If you wish to make use of more fine-grained weather data, it will likely require more advanced model creation and MTG manipulation, and more involved work on the modeling side. + +### Organ/Scale + +Plants have different organs with distinct physiological properties and processes. When doing more fine-grained simulations of plant growth, many models will be tied to a particular organ of a plant. Models handling flowering state or root water absorption are such examples. Others, such as carbon allocation and demand, might be reused in slightly different ways for multiple organs of the plant. + +PlantSimEngine documentation tends to use the terms "organ" and "scale" mostly interchangeably. "Scale" is a bit more general and accurate, since some models might not operate at a specific organ level, but (for example) at the scene level, so a "Scene" scale might be present in the MTG, and in the user-provided data. + +When working with multi-scale data, the scale will often need to be specified to map variables, or to indicate at what scale level models work out. You will see some code resembling this : + +```julia +"Root" => (RootGrowthModel(), OrganAgeModel()), +"Leaf" => (LightInterceptionModel(), OrganAgeModel()), +"Plant" => (TotalBiomassModel(),), +``` + +This example excerpt links from specific models to a specific scale. Note that one model is reused at two different scales, and note that "Plant" isn't an actual organ, hence the preferred usage of the term "scale". + +### Multiscale modeling + +Multi-scale modeling is the process of simulating a system at multiple levels of detail simultaneously. Some models might run at the organ scale while others run at the plot scale. Each model can access variables at its scale and other scales if needed, allowing for a more comprehensive system representation. It can also help identify emergent properties that are not apparent at a single level of detail. + +For example, a model of photosynthesis at the leaf scale can be combined with a model of carbon allocation at the plant scale to simulate the growth and development of the plant. Another example is a combination of models to simulate the energy balance of a forest. To simulate it, you need a model for each organ type of the plant, another for the soil, and finally, one at the plot scale, integrating all others. + +When running multi-scale simulations which contain models operating at different organ levels for the plant, extra information needs to be provided by the user to run models. Since some models are reused at different organ levels, it is necessary to indicate which organ level a model operates at. + +This is why multi-scale simulations make use of a 'mapping': the ModelList in the single-scale examples does not have a way to tie models to plant organs, and the more versatile models could be used in various places. The user must also indicate how models operate with other scales, *e.g.* if an input variable comes from another scale, it is required to indicate which scale it is mapped from. + +You can read more about some practical differences as a user between single- and multi-scale simulations here: [Multi-scale considerations](@ref). + +!!! note + When you encounter the terms "Single-scale simulations", or "ModelList simulations", they will refer to simulations that are "not multi-scale". A multi-scale simulation makes use of a mapping between different organ/scale levels. A single-scale simulation has no such mapping, and uses the simpler ModelList interface. + You can implement a mapping that only makes use of a single scale level, of course, making it a "single-scale multi-scale simulation", but **unless otherwise specified, single-scale, and the whole section dedicated to single-scale simulations, refer to simulations with ModelList objects, and no mapping**. + +### Multi-scale Tree Graphs + +![Grassy plant and equivalent MTG](../www/Grassy_plant_MTG_vertical.svg) + +A Grassy plant and its equivalent MTG + +Multi-scale Tree Graphs (MTG) are a data structure used to represent plants. A more detailed introduction to the format and its attributes can be found [in the MultiScaleTreeGraph.jl package documentation](https://vezy.github.io/MultiScaleTreeGraph.jl/stable/the_mtg/mtg_concept/). + +Multi-scale simulations can operate on MTG objects; new nodes are added corresponding to new organs created during the plant's growth. + +You can see a basic display of an MTG by simply typing its name in the REPL: + +![example display of an MTG in PlantSimEngine](../www/MTG_output.png) + +!!! note + Another companion package, [PlantGeom.jl](https://github.com/VEZY/PlantGeom.jl), can also create MTG objects from .opf files (corresponding to the [Open Plant Format](https://amap-dev.cirad.fr/projects/xplo/wiki/The_opf_format_(*opf)), an alternate means of describing plants computationally). + +#### Scale/symbol terminology ambiguity + +Multi-scale tree graphs have different terminology (see [Organ/Scale](@ref)): + +- the MTG node **symbol** represents "something" like a "Plant", "Root", "Scene" or "Leaf". It corresponds to a PlantSimEngine *scale* and has nothing to do with the Julia programming language's definition of symbol (*e.g.* `:var`) +- the MTG node **scale**, is an integer passed to the Node constructor, and describes the level of description of the tree graph object. They don't always have a one-to-one correspondence to the symbol (or PlantSimEngine's scale), but are similar. + +![Three scale levels on an MTG, which differ from typical PlantSimEngine concept of scale](../www/Grassy_plant_scales.svg) + +You can find a brief description of the MTG concepts [here](https://vezy.github.io/MultiScaleTreeGraph.jl/stable/the_mtg/mtg_concept/#Node-MTG-and-attributes). + +Other words are unfortunately reused in various contexts with different meanings: tree/leaf/root have a different meaning when talking about computer science data structure (*e.g.*, graphs, dependency graphs and trees). + +!!! note + In the majority of cases, you can assume the tree-related terminology refers to the biological terms, and that "organ" refer to plant organs, and "single-scale", "multi-scale" and "scale" to PlantSimEngine's concept of scales described in [Organ/Scale](@ref). MTG objects are mostly manipulated on a per-node basis (the graph node, not the botanical node), unless a model makes use of functions relating to MTG traversal, in which case you may expect computer science terminology. + +#### TLDR + +In summary: + +- In PlantSimEngine a scale is a level of description defined by a name (`String`). In MTG, a scale is an integer describing the level of description of the node, and a symbol is a name for that node. So symbol in the MTG == scale in PlantSimEngine; +- The word "node" is always used to refer to the Multiscale Tree Graph node, not the botanical node. + +### State machines + +A state machine is a computational concept used to model mechanisms and devices, which may be of interest for your simulations. + +![State machine image](../www/Turnstile_state_machine_colored.svg.png) +A simple state machine. See the [wikipedia page](https://en.wikipedia.org/wiki/Finite-state_machine) for more examples. + +State machines can be useful to model organ state: some organs in [XPalm.jl](https://github.com/PalmStudio/XPalm.jl), a package modelling the oil palm using PlantSimEngine, have a `state` variable behaving like a state machine, indicating whether an organ is mature, pruned, flowering, etc. + +You can find an example model (amongst other such models) affecting the `state` variable of some organs depending on their age and thermal time in the XPalm oil palm FSPM [here](https://github.com/PalmStudio/XPalm.jl/blob/main/src/plant/phytomer/phytomer/state.jl). diff --git a/docs/src/step_by_step/advanced_coupling.md b/docs/src/step_by_step/advanced_coupling.md new file mode 100644 index 000000000..36cad842b --- /dev/null +++ b/docs/src/step_by_step/advanced_coupling.md @@ -0,0 +1,64 @@ +# Coupling more complex models + +```@setup usepkg +using PlantSimEngine, PlantMeteo +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples + +m = ModelList( + Process1Model(2.0), + Process2Model(), + Process3Model(), + Process4Model(), + Process5Model(), + Process6Model(), + Process7Model(), +) +``` + +When two or more models have a two-way interdependency (rather than variables flowing out only one-way from one model into the next), we describe it as a [hard dependency](@ref hard_dependency_def). + +This kind of interdependency requires a little more work from the user/modeler for PlantSimEngine to be able to automatically create the dependency graph. + +## Declaring hard dependencies + +A model that explicitly and directly calls another process in its [`run!`](@ref) function is part of a hard dependency, or a hard-coupled model. + +Let's go through the example processes and models from a script provided by the package here [examples/dummy.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/dummy.jl) + +In this script, we declare seven processes and seven models, one for each process. The processes are simply called "process1", "process2"..., and the model implementations are called `Process1Model`, `Process2Model`... + +When run, `Process2Model` calls another process's [`run!`](@ref) function explicitely, which requires defining that process as a hard-dependency of `Process2Model` : + +```julia +function PlantSimEngine.run!(::Process2Model, models, status, meteo, constants, extra) + # computing var3 using process1: + run!(models.process1, models, status, meteo, constants) + # computing var4 and var5: + status.var4 = status.var3 * 2.0 + status.var5 = status.var4 + 1.0 * meteo.T + 2.0 * meteo.Wind + 3.0 * meteo.Rh +end +``` + +`Process2Model` is coupled to another process (`process1`), and calls its model's `run` function. The [`run!`](@ref) function is called with the same arguments as the [`run!`](@ref) function of the model that calls it, except that we pass the process we want to simulate as the first argument. + +!!! note + We don't enforce any type of model to simulate `process1`. This is the reason why we can switch so easily between model implementations for any process, by just changing the model in the [`ModelList`](@ref). + +A hard-dependency must always be declared to PlantSimEngine. This is done by adding a method to the `dep` function when implementing the model. For example, the hard-dependency to `process1` into `Process2Model` is declared as follows: + +```julia +PlantSimEngine.dep(::Process2Model) = (process1=AbstractProcess1Model,) +``` + +This way PlantSimEngine knows that `Process2Model` needs a model for the simulation of the `process1` process. To avoid imposing a specific model to be coupled with `Process2Model`, the dependency only requires a model that is a subtype of the abstract parent type `AbstractProcess1Model`. This avoids constraining to the specific `Process1Model` implementation, meaning an alternate model computing the same variables for the same process is still interchangeable with `Process1Model`. + +While not encouraged, if you have a valid reason to force the coupling with a particular model, you can force the dependency to require that model specifically. For example, if we want to use only `Process1Model` for the simulation of `process1`, we would declare the dependency as follows: + +```julia +PlantSimEngine.dep(::Process2Model) = (process1=Process1Model,) +``` + +## Examples in the wild + +You can find a typical example in a companion package: [PlantBioPhysics.jl](https://github.com/VEZY/PlantBiophysics.jl). An energy balance model, the [Monteith model](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl), needs to [iteratively run a photosynthesis model](https://github.com/VEZY/PlantBiophysics.jl/blob/c1a75f294109d52dc619f764ce51c6ca1ea897e8/src/processes/energy/Monteith.jl#L154) in its [`run!`](@ref) function. \ No newline at end of file diff --git a/docs/src/step_by_step/detailed_first_example.md b/docs/src/step_by_step/detailed_first_example.md new file mode 100644 index 000000000..6e38cca19 --- /dev/null +++ b/docs/src/step_by_step/detailed_first_example.md @@ -0,0 +1,245 @@ +# [Detailed walkthrough of a simple simulation](@id detailed-walkthrough-of-a-simple-simulation) + +This page walks you through the ins and outs of a basic simulation, mostly aimed at people who have less experience programming, to showcase the various concepts presented earlier and requirements for a simulation in context. + +A working trimmed-down script can be found further down in the [Example simulation](@ref), and other subsections in this page will detail setup and helper functions, and querying outputs. + +If you simply wish to copy-paste examples and tinker with them, you can find a few examples on the [Quick examples](@ref) page. + +```@setup usepkg +using PlantSimEngine, PlantMeteo +using PlantSimEngine.Examples +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) +leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) +out_sim = run!(leaf, meteo) +``` + +```@contents +Pages = ["detailed_first_example.md"] +Depth = 3 +``` + +## Setting up your environment + +For every script in this documentation, you will always need a working Julia environment with PlantSimengine added to it, and usually several other companion packages. Details for getting to that point are provided on the [Installing and running PlantSimEngine](@ref) page. + +## Definitions + +### Processes + +A process in this package defines a biological or physical phenomena. Think of any process happening in a system, such as light interception, photosynthesis, water, carbon and energy fluxes, growth, yield or even electricity produced by solar panels. + +A process is "declared", meaning we define a process, and then implement models for its simulation. In this example, we will make use of a process that was already defined, and for which there already is a model implementation. + +### Models (ModelList) + +A process is simulated using a particular implementation, or **a model**. Each model is implemented using a structure that lists the parameters of the model. For example, PlantBiophysics provides the [`Beer`](https://vezy.github.io/PlantBiophysics.jl/stable/functions/#PlantBiophysics.Beer) structure for the implementation of the Beer-Lambert law of light extinction. The process of `light_interception` and the `Beer` model are provided as an example +script in this package too at [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/Beer.jl). + +Models can use several types of entries: + +- Parameters +- Meteorological information +- Variables +- Constants +- Extras + +**Parameters** are constant values that are used by the model to compute its outputs, and are exclusive to that model. + +**Meteorological information** contains values that are provided by the user and are used as inputs to the model. It is defined for one time-step, and `PlantSimEngine.jl` takes care of applying the model to each time-steps given by the user. + +**Variables** are either used or computed by the model and can optionally be initialized before the simulation. They can be part of multiple models, computed by one and then used as an input by another. They can also be a global simulation output, or be provided at the start of a simulation by the user. + +**Constants** are constant values, usually common between models, *e.g.* the universal gas constant. + +And **extras** are just extra values that can be used by a model, or serves as a placeholder for internal data. + +Users declare a set of models used for simulation, as well as the necessary parameters for each model, and whatever variables need to be initialized. This is done using a [`ModelList`](@ref) structure. + +For example let's instantiate a [`ModelList`](@ref) with a single model : the Beer-Lambert model of light extinction, used to simulate the light interception process. The model is implemented with the [`Beer`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/Beer.jl) structure and only has one parameter: the extinction coefficient (`k`). + +Importing the package: + +```@example usepkg +using PlantSimEngine +``` + +Import the examples defined in the [`Examples`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples) sub-module (`light_interception` and `Beer`): + +```julia +using PlantSimEngine.Examples +``` + +And then declare a [`ModelList`](@ref) with the `Beer` model: + +```@example usepkg +m = ModelList(Beer(0.5)) +``` + +What happened here? We provided an instance of the `Beer` model to a [`ModelList`](@ref) to simulate the light interception process. + +## Parameters + +A parameter is a value constant for a simulation that is internal to a model and used for its computations. For example, the Beer-Lambert model uses the extinction coefficient (`k`) to compute the light extinction. The `Beer` structure in the Beer-Lambert model implementation, only has one field: `k`. We can see that using `fieldnames` on the model structure: + +```@example usepkg +fieldnames(Beer) +``` + +## Variables (inputs, outputs) + +Variables are either inputs or outputs (*i.e.* computed) of models. Variables and their values are stored in the [`ModelList`](@ref) structure, and are initialized automatically or manually. + +For example, the `Beer` model needs the leaf area index (`LAI`, m² m⁻²) to run. + +We can see which variables are passed in as inputs using [`inputs`](@ref): + +```@example usepkg +inputs(Beer(0.5)) +``` + +and which are computed outputs of the model using [`outputs`](@ref): + +```@example usepkg +outputs(Beer(0.5)) +``` + +The [`ModelList`](@ref) structure will keep track of every variable's current state when running the simulation, storing them in a field called `status`. We can inspect that field with the [`status`](@ref) function and see that in our example it has two variables: `LAI` and `PPFD`. The first is an input, the second an output (*i.e.* it is computed by the model). + +```@example usepkg +m = ModelList(Beer(0.5)) +keys(status(m)) +``` + +To know which variables should be initialized, we can use [`to_initialize`](@ref): + +```@example usepkg +m = ModelList(Beer(0.5)) +to_initialize(m) +``` + +Their values are uninitialized though (hence the warnings): + +```@example usepkg +(m[:LAI], m[:aPPFD]) +``` + +Uninitialized variables are initialized to the value given in the [`inputs`](@ref) or [`outputs`](@ref) methods in the model's implementation code, which is usually equal to `typemin()`, *e.g.* `-Inf` for `Float64`. + +!!! tip + Prefer using [`to_initialize`](@ref) rather than [`inputs`](@ref) to check which variables should be initialized. [`inputs`](@ref) returns every variable that is needed by the model to run, but in multi-model simulations, some of them may already be computed by other models and not require initialization. [`to_initialize`](@ref) returns **only** the variables that are needed by the model to run and that are not initialized in the [`ModelList`](@ref). + +We can initialize the required variables by providing their starting values to the status when declaring the `ModelList`: + +```@example usepkg +m = ModelList(Beer(0.5), status = (LAI = 2.0,)) +``` + +Or after instantiation using [`init_status!`](@ref): + +```@example usepkg +m = ModelList(Beer(0.5)) + +init_status!(m, LAI = 2.0) +``` + +We can check if a component is correctly initialized using [`is_initialized`](@ref): + +```@example usepkg +is_initialized(m) +``` + +Some variables are inputs of models, but outputs of other models. When we couple models, [`to_initialize`](@ref) only requests the variables that are not computed by other models. + +## Climate forcing + +To make a simulation, we usually need the climatic/meteorological conditions measured close to the object or component. + +Users are strongly encouraged to use [`PlantMeteo.jl`](https://github.com/PalmStudio/PlantMeteo.jl), the companion package that helps manage such data, with default pre-computations and structures for efficient computations. The most basic data structure from this package is a type called [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere), which defines steady-state atmospheric conditions, *i.e.* the conditions are considered at equilibrium. Another structure is available to define different consecutive time-steps: [`TimeStepTable`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.TimeStepTable). + +The mandatory variables to provide for an [`Atmosphere`](https://palmstudio.github.io/PlantMeteo.jl/stable/#PlantMeteo.Atmosphere) are: `T` (air temperature in °C), `Rh` (relative humidity, 0-1) and `Wind` (the wind speed, m s⁻¹). In our example, we also need the incoming photosynthetically active radiation flux (`Ri_PAR_f`, W m⁻²). We can declare such conditions like so: + +```@example usepkg +using PlantMeteo +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) +``` + +This `meteo` variable will therefore provide a single weather timeframe that can be used in a simulation. + +More details are available from the [package documentation](https://vezy.github.io/PlantMeteo.jl/stable). + +## Simulation + +### Simulation of processes + +To run a simulation, you can call the [`run!`](@ref) method on the [`ModelList`](@ref). If some meteorological data is required for models to be simulated over several timesteps, that can be passed in as an optional argument as well. + +Your call to the function would then look like this: + +```julia +run!(model_list, meteo) +``` + +The first argument is the model list (see [`ModelList`](@ref)), and the second defines the micro-climatic conditions. + +The [`ModelList`](@ref) should already be initialized for the given process before calling the function. Refer to the earlier subsection [Variables (inputs, outputs)](@ref) for more details. + +### Example simulation + +For example we can simulate the `light_interception` of a leaf like so: + +```@example usepkg +using PlantSimEngine, PlantMeteo + +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples + +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) + +leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) + +outputs_example = run!(leaf, meteo) + +outputs_example[:aPPFD] +``` + +### Outputs + +The [`status`](@ref) field of a [`ModelList`](@ref) is used to initialize the variables before simulation and then to keep track of their values during and after the simulation. We can extract outputs of the very last timestep of a simulation using the [`status`](@ref) function. + +The actual full output data is returned by the [`run!`](@ref) function. Data is usually stored in a [`TimeStepTable`](@ref) structure from `PlantMeteo.jl`, which is a fast DataFrame-like structure with each time step being a [`Status`](@ref). It can be also be any `Tables.jl` structure, such as a regular `DataFrame`. The weather is also usually stored in a [`TimeStepTable`](@ref) but with each time step being an `Atmosphere`. + +In our example, the simulation was only provided one weather timestep, so the outputs returned by [`run!`](@ref) and the ModelList's [`status`](@ref) field are identical. + +Let's look at the outputs structure of our previous simulated leaf: + +```@setup usepkg +outputs_example +``` + +We can extract the value of one variable by indexing into it, *e.g.* for the intercepted light: + +```@example usepkg +outputs_example[:aPPFD] +``` + +Or similarly using the dot syntax: + +```@example usepkg +outputs_example.aPPFD +``` + +You can then print the outputs, convert them to another format, or visualize them, using other Julia packages. You can read more on how to do that in the [Visualizing outputs and data](@ref) page. + +Another convenient way to get the results is to transform the outputs into a `DataFrame`. Which is very easy because the [`TimeStepTable`](@ref) implements the Tables.jl interface: + +```@example usepkg +using DataFrames +convert_outputs(outputs_example, DataFrame) +``` + +## Model coupling + +A model can work either independently or in conjunction with other models. For example a stomatal conductance model is often associated with a photosynthesis model, *i.e.* it is called from the photosynthesis model. + +`PlantSimEngine.jl` is designed to make model coupling painless for modelers and users. Please see [Standard model coupling](@ref) and [Coupling more complex models](@ref) for more details, or [Handling dependencies in a multiscale context](@ref) for multi-scale specific coupling considerations. diff --git a/docs/src/step_by_step/implement_a_model.md b/docs/src/step_by_step/implement_a_model.md new file mode 100644 index 000000000..5cc143a43 --- /dev/null +++ b/docs/src/step_by_step/implement_a_model.md @@ -0,0 +1,248 @@ +# [Implementing a model](@id model_implementation_page) + +```@setup usepkg +using PlantSimEngine +@process "light_interception" verbose = false +struct Beer{T} <: AbstractLight_InterceptionModel + k::T +end +``` + +For your own simulations, you might want to move beyond simple usage at some point and implement your own models. In this page, we'll go through the required steps for writing a new model. The detailed version is tailored for people less familiar with programming. + +## Quick version + +Declare a new process : + +```julia +@process "light_interception" verbose = false +``` + +Declare your model struct, and its parameters : + +```@example usepkg +struct Beer{T} <: AbstractLight_InterceptionModel + k::T +end +``` + +Declare the `inputs_` and `outputs_` methods for that model (note the '_', these methods are distinct from `inputs` and `outputs`) + +```@example usepkg +function PlantSimEngine.inputs_(::Beer) + (LAI=-Inf,) +end + +function PlantSimEngine.outputs_(::Beer) + (aPPFD=-Inf,) +end +``` + +Write the [`run!`](@ref) function that operates on a single timestep : + +```@example usepkg +function run!(::Beer, models, status, meteo, constants, extras) + status.PPFD = + meteo.Ri_PAR_f * + exp(-models.light_interception.k * status.LAI) * + constants.J_to_umol +end +``` + +Determine if parallelization is possible, and which traits to declare : + +```@example usepkg +PlantSimEngine.ObjectDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsTimeStepIndependent() +``` + +And that is all you need to get going, for this example with a single parameter and no interdependencies. + +The [`@process`](@ref) macro does some boilerplate work described [here](@ref under_the_hood) + +Some extra utility functions can also be interesting to implement to make users' lives simpler. See the [Model implementation additional notes](@ref) page for details. +If your custom model needs to handle more complex couplings than the simple input/output described in this example, check out the [Coupling more complex models](@ref) page. + +## Detailed version + +`PlantSimEngine.jl` was designed to make new model implementation very simple. So let's learn about how to implement your own model with a simple example: implementing a new light interception model. + +The model we'll (re)implement is available as an example model from the `Examples` sub-module. You can access the script from here: [`examples/Beer.jl`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/Beer.jl). It is also available in the `PlantBioPhysics.jl` package. + +You can import the model and PlantSimEngine's other example models into your environment with `using`: + +```julia +# Import the example models defined in the `Examples` sub-module: +using PlantSimEngine.Examples +``` + +## Other examples + +`PlantSimEngine`'s other toy models can be found in the [examples](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples) folder. + +For other examples, you can look at the code in [`PlantBiophysics.jl`](https://github.com/VEZY/PlantBiophysics.jl), where you will find *e.g.* a photosynthesis model, with the implementation of the `FvCB` model in [src/photosynthesis/FvCB.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/photosynthesis/FvCB.jl); an energy balance model with the implementation of the `Monteith` model in [src/energy/Monteith.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/energy/Monteith.jl); or a stomatal conductance model in [src/conductances/stomatal/medlyn.jl](https://github.com/VEZY/PlantBiophysics.jl/blob/master/src/processes/conductances/stomatal/medlyn.jl). + +## Requirements + +If you have a look at example models, you'll see that in order to implement a new model you'll need to implement: + +- a structure, used to hold the parameter values and to dispatch to the right method +- the actual model, developed as a method for the process it simulates +- some helper functions used by the package and/or the users + +## Example: the Beer-Lambert model + +### The process + +We start by declaring the light interception process at l.7 using [`@process`](@ref): + +```julia +@process "light_interception" verbose = false +``` + +See [Implementing a new process](@ref) for more details on how that works and how to use the process. + +### The structure + +To implement a model, the first thing to do is to define a structure. The purpose of this structure is two-fold: + +- hold the parameter values +- dispatch to the right [`run!`](@ref) method when calling it + +The structure of the model (or type) is defined as follows: + +```@example usepkg +struct Beer{T} <: AbstractLight_InterceptionModel + k::T +end +``` + +The first line defines the name of the model (`Beer`). It is good practice to use camel case for the name, *i.e.* using capital letters for the words and no separator `LikeThis`. + +The `Beer` structure is defined as a subtype of `AbstractLight_InterceptionModel` indicating what kind of process the model simulates. The `AbstractLight_InterceptionModel` type is automatically created when defining the process "light_interception". + +We can therefore infer from the declaration that `Beer` is a model to simulate the light interception process. + +Then come the parameters names, and their types. + +### User types and parametric types + +There is a little Julia specificity here, to enable the user to pass their own types to the simulation. + +- `Beer` is a parameterized `struct`, indicated by the `{T}` annotation +- We indicate the `k` parameter is of type `T` by adding `::T` after the name. + +The `T` is an arbitrary letter here. If you have parameters that you know will be of different types, you can either force their type, or make them parameterizable too, using another letter, *e.g.*: + +```julia +struct CustomModel{T,S} <: AbstractLight_InterceptionModel + k::T + x::T + y::T + z::S +end +``` + +Parameterized types are practical because they let the user choose the type of the parameters, and potentially change them at runtime. For example a user could use the `Particles` type from [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) for automatic uncertainty propagation throughout the simulation. We refer you to the [Parametric types](@ref) subsection of the [Model implementation additional notes](@ref) page for more information on parametric types. + +### Inputs and outputs + +When implementing a new model, it is necessary to declare what variables will be required, whether provided as an input to our model or computed for every timestep as an output. Input variables will either be initialized by the user in a `Status` object, or provided by another model. Output variables may be global simulation outputs and/or used by other models. + +In our case, the `Beer` model, computing light interception, has one input variable and one output variable: + +- Inputs: `:LAI`, the leaf area index (m² m⁻²) +- Outputs: `:aPPFD`, the photosynthetic photon flux density (μmol m⁻² s⁻¹) + +We declare these inputs/outputs by adding a method for the [`inputs`](@ref) and [`outputs`](@ref) functions. These functions take the type of the model as argument, and return a `NamedTuple` with the names of the variables as keys, and their default values as values: + +```@example usepkg +function PlantSimEngine.inputs_(::Beer) + (LAI=-Inf,) +end + +function PlantSimEngine.outputs_(::Beer) + (aPPFD=-Inf,) +end +``` + +These functions are internal, and end with an "\_". Users instead use [`inputs`](@ref) and [`outputs`](@ref) to query model variables. + +### The run! method + +When running a simulation with [`run!`](@ref), each model is run in turn at every timestep, following whatever order was deduced from the ModelList definition and Status. Each model also has its [`run!`](@ref) method for that purpose that update the simulation's current state, with a slightly different signature. The function takes six arguments: + +```julia +function run!(::Beer, models, status, meteo, constants, extras) +``` + +- the model's type +- models: a [`ModelList`](@ref) object, which contains all the models of the simulation +- status: a [`Status`](@ref) object, which contains the current values (*i.e.* state) of the variables for **one** time-step (e.g. the value of the plant LAI at time t) +- meteo: (usually) an `Atmosphere` object, or a row of the meteorological data, which contains the current values of the meteorological variables for **one** time-step (*e.g.* the value of the PAR at time t) +- constants: a `Constants` object, or a `NamedTuple`, which contains the values of the constants for the simulation (*e.g.* the value of the Stefan-Boltzmann constant, unit-conversion constants...) +- extras: any other object you want to pass to your model, mostly for advanced usage, not detailed here + +A typical [`run!`](@ref) function can therefore make use of simulation constants, input/output variables accessible through the [`Status`](@ref object, or weather data. + +Here is the [`run!`](@ref) implementation of the light interception for a [`ModelList`](@ref) component models. Note that the input and output variable are accessed through the [`status`](@ref) argument : + +```@example usepkg +function run!(::Beer, models, status, meteo, constants, extras) + status.PPFD = + meteo.Ri_PAR_f * + exp(-models.light_interception.k * status.LAI) * + constants.J_to_umol +end +``` + +### Additional notes + +To use this model, users will have to make sure that the variables for that model are defined in the [`Status`](@ref) object, the meteorology, and the `Constants` object. + +!!! Note + [`Status`](@ref) objects contain the current state of the simulation. It is not, by default, possible to make use of earlier variable states, unless a custom model is written for that purpose. + +Model parameters are available from the [`ModelList`](@ref) that is passed via the `models` argument. Index by the process name, then the parameter name. For example, the `k` parameter of the `Beer` model is found in `models.light_interception.k`. + +!!! warning + You need to import all the functions you want to extend, so Julia knows your intention of adding a method to the function from PlantSimEngine, and not defining your own function. To do so, you have to prefix the said functions by the package name, or import them before *e.g.*: `import PlantSimEngine: inputs_, outputs_`. The troubleshooting subsection [Implementing a model: forgetting to import or prefix functions](@ref) showcases output errors that can occur when you forget to prefix. + +### Parallelization traits + +`PlantSimEngine` defines traits to get additional information about the models. At the moment, there are two traits implemented that help the package to know if a model can be run in parallel over space (*i.e.* objects) and/or time (*i.e.* time-steps). + +By default, all models are assumed to be **not** parallelizable over objects and time-steps, because it is the safest default. If your model is parallelizable, you should add the trait to the model. + +For example, if we want to add the trait for parallelization over objects to our `Beer` model, we would do: + +```@example usepkg +PlantSimEngine.ObjectDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsObjectIndependent() +``` + +And if we want to add the trait for parallelization over time-steps to our `Beer` model, we would do: + +```@example usepkg +PlantSimEngine.TimeStepDependencyTrait(::Type{<:Beer}) = PlantSimEngine.IsTimeStepIndependent() +``` + +!!! note + A model is parallelizable over objects if it does not call another model directly inside its code. Similarly, a model is parallelizable over time-steps if it does not get values from other time-steps directly inside its code. In practice, most of the models are parallelizable one way or another, but it is safer to assume they are not. + +OK that's it! We now a full new model implementation for the light interception process! Other models might be more complex in terms of what computations they do, or how they couple with other models, but the approach remains the same. + +### Dependencies + +If your model explicitly calls another model, you need to tell PlantSimEngine about it. This is called a hard dependency, in opposition to a soft dependency, which is when your model uses a variable from another model, but does not call it explicitly. + +To do so, we can add a method to the [`dep`](@ref) function that tells PlantSimEngine which processes (and models) are needed for the model to run. + +Our example model does not call another model, so we don't need to implement it. But we can look at *e.g.* the implementation for [`Fvcb`](https://github.com/VEZY/PlantBiophysics.jl/blob/d1d5addccbab45688a6c3797e650a640209b8359/src/processes/photosynthesis/FvCB.jl#L83) in `PlantBiophysics.jl` to see how it works: + +```julia +PlantSimEngine.dep(::Fvcb) = (stomatal_conductance=AbstractStomatal_ConductanceModel,) +``` + +Here we say to PlantSimEngine that the `Fvcb` model needs a model of type `AbstractStomatal_ConductanceModel` in the stomatal conductance process. + +You can read more about hard dependencies in [Coupling more complex models](@ref). diff --git a/docs/src/step_by_step/implement_a_model_additional.md b/docs/src/step_by_step/implement_a_model_additional.md new file mode 100644 index 000000000..b3f943669 --- /dev/null +++ b/docs/src/step_by_step/implement_a_model_additional.md @@ -0,0 +1,101 @@ +# Model implementation additional notes + +```@contents +Pages = ["implement_a_model_additional.md"] +Depth = 3 +``` + +## Parametric types + +In [Implementing a model](@ref model_implementation_page), the Beer model's structure was declared with a parametric type. + +```julia +struct Beer{T} <: AbstractLight_InterceptionModel + k::T +end +``` + +Why not force the type ? Float64 is more accurate than Float32, after all: + +```julia +struct YourStruct <: AbstractLight_InterceptionModel + k::Float64 + x::Float64 + y::Float64 + z::Int +end +``` + +Doing so would lose some flexibility in the way users can make use of your models. For example a user could use the `Particles` type from [MonteCarloMeasurements.jl](https://github.com/baggepinnen/MonteCarloMeasurements.jl) for automatic uncertainty propagation, and this is only possible if the model type is parameterizable. Forcing a `Float64` type would render the model incompatible with `Particles`. + +## Type promotion + +When implementing a new model, you can do a little optional extra work to help future users. + +You can add a method for type promotion. It wouldn't make any sense for the previous `Beer` example because we have only one parameter. But we can make another example with a new model that would be called `Beer2` that would take two parameters: + +```julia +struct Beer2{T} <: AbstractLight_InterceptionModel + k::T + x::T +end +``` + +To add type promotion to `Beer2` we would do: + +```julia +function Beer2(k,x) + Beer2(promote(k,x)...) +end +``` + +!!! note + `promote` returns a NamedTuple, which needs to be splatted for the constructor, see the [Julia docs](https://docs.julialang.org/en/v1/manual/conversion-and-promotion/#Promotion) for a more in-depth explanation, or our [Getting started with Julia](@ref) page for some links to other references discussing Julia concepts used in PlantSimEngine. + +This would allow users to instantiate the model parameters using different types of inputs. For example users may write the following: + +```julia +Beer2(0.6,2) +``` + +`Beer2` is a parametric type, with all fields sharing the same type `T`. This is the `T` in `Beer2{T}` and then in `k::T` and `x::T`. And this forces the user to give all parameters with the same type. + +And in the example above, providin `0.6` for `k`, which is a `Float64`, and `2` for `x`, which is an `Int`. If you don't have type promotion, Julia will return an error because both should be either `Float64` or `Int`. That's were type promotion comes in handy, as it will convert all your inputs to a common type (when possible). In our case it will convert `2` to `2.0`. + +## Other helper functions and constructors + +### Default parameter values + +You can simplify model usage by helping your user with default values for some parameters (if applicable). For example, in the `Beer` model a user will almost never change the value of `k`. So we can provide a default value like so: + +```@example usepkg +Beer() = Beer(0.6) +``` + +Now the user can call `Beer` with no arguments, and `k` will default to `0.6`. + +### Parameter values as kwargs + +Another useful thing is the ability to instantiate your model type with keyword arguments, *i.e.* naming the arguments. You can do it by adding the following method: + +```@example usepkg +Beer(;k) = Beer(k) +``` + +The `;` syntax indicates that subsequent arguments are provided as keyword arguments, so now we can call `Beer` like this: + +```julia +Beer(k = 0.7) +``` + +This helps readability when there are a lot of parameters and some have default values. + +### eltype + +The last optional utility function to implement is a method for the `eltype` function: + +```julia +Base.eltype(x::Beer{T}) where {T} = T +``` + +This one helps Julia know the type of the elements in the structure, and make it faster. \ No newline at end of file diff --git a/docs/src/step_by_step/implement_a_process.md b/docs/src/step_by_step/implement_a_process.md new file mode 100644 index 000000000..74142ce19 --- /dev/null +++ b/docs/src/step_by_step/implement_a_process.md @@ -0,0 +1,59 @@ +# Implementing a new process + +```@setup usepkg +using PlantSimEngine +using PlantMeteo +PlantSimEngine.@process growth +``` + +## Introduction + +A process in this package defines a biological or physical phenomena. Think of any process happening in a system, such as light interception, photosynthesis, water, carbon and energy fluxes, growth, yield or even electricity produced by solar panels. + +`PlantSimEngine.jl` was designed to make the implementation of new processes and models easy and fast. The next section showcases how to implement a new process with a simple example: implementing a growth model. + +## Implement a process + +A process is "declared", meaning we define a process, and then implement models for its simulation. Declaring a process generates some boilerplate code for its simulation: + +- an abstract type for the process +- a method for the `process` function, that is used internally + +The abstract process type is then used as a supertype of all models implementations for the process, and is named `AbstractProcess`, *e.g.* `AbstractLight_InterceptionModel`. + +Fortunately, PlantSimEngine provides a macro to generate all that at once: [`@process`](@ref). This macro takes only one argument: the name of the process. + +For example, the photosynthesis process in [PlantBiophysics.jl](https://github.com/VEZY/PlantBiophysics.jl) is declared using just this tiny line of code: + +```julia +@process "photosynthesis" +``` + +If we want to simulate the growth of a plant, we could add a new process called `growth`: + +```julia +@process "growth" +``` + +And that's it! Note that the function guides you in the steps you can make after creating a process. + +## Implement a new model for the process + +Once process implementation is done, you can write a corresponding model implementation. A tutorial page showcasing a light interception model implementation can be found [here](@ref model_implementation_page) + +A full model implementation for this process is available in the example script [ToyAssimGrowthModel.jl](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/examples/ToyAssimGrowthModel.jl). + +## [Under the hood](@id under_the_hood) + +The `@process` macro is just a shorthand reducing boilerplate. + +You can in its stead directly define a process by hand by defining an abstract type that is a subtype of `AbstractModel`: +```julia +abstract type AbstractGrowthModel <: PlantSimEngine.AbstractModel end +``` +And by adding a method for the `process_` function that returns the name of the process: +```julia +PlantSimEngine.process_(::Type{AbstractGrowthModel}) = :growth +``` + +So in the earlier example, a new process was created called `growth`. This defined a new abstract structure called `AbstractGrowthModel`, which is used as a supertype of the models. This abstract type is always named using the process name in title case (using `titlecase()`), prefixed with `Abstract` and suffixed with `Model`. \ No newline at end of file diff --git a/docs/src/step_by_step/model_switching.md b/docs/src/step_by_step/model_switching.md new file mode 100644 index 000000000..4442007b0 --- /dev/null +++ b/docs/src/step_by_step/model_switching.md @@ -0,0 +1,100 @@ +# Model switching + +```@setup usepkg +using PlantSimEngine, PlantMeteo, CSV, DataFrames +# Import the examples defined in the `Examples` sub-module +using PlantSimEngine.Examples + +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) +run!(models, meteo_day) +models2 = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyAssimGrowthModel(), + status=(TT_cu=cumsum(meteo_day.TT),), +) +run!(models2, meteo_day) +``` + +One of the main objective of PlantSimEngine is allowing users to switch between model implementations for a given process **without making any change to the PlantSimEngine codebase**. + +The package was designed around this idea to make easy changes easy and efficient. Switch models in the [`ModelList`](@ref), and call the [`run!`](@ref) function again. No other changes are required if no new variables are introduced. + +## A first simulation as a starting point + +With a working environment, let's create a [`ModelList`](@ref) with several models from the example scripts in the [`examples`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/) folder: + +Importing the models from the scripts: + +```julia +using PlantSimEngine +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples +``` + +Coupling the models in a [`ModelList`](@ref): + +```@example usepkg +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +nothing # hide +``` + +We can the simulation by calling the [`run!`](@ref) function with meteorology data. Here we use an example data set: + +```@example usepkg +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +nothing # hide +``` + +We can now run the simulation: + +```@example usepkg +output_initial = run!(models, meteo_day) +``` + +## Switching one model in the simulation + +Now what if we want to switch the model that computes growth ? We can do this by simply replacing the model in the [`ModelList`](@ref), and PlantSimEngine will automatically update the dependency graph, and adapt the simulation to the new model. + +Let's switch ToyRUEGrowthModel with ToyAssimGrowthModel: + +```@example usepkg +models2 = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyAssimGrowthModel(), # This was `ToyRUEGrowthModel(0.2)` before + status=(TT_cu=cumsum(meteo_day.TT),), +) + +nothing # hide +``` + +ToyAssimGrowthModel is a little bit more complex than `ToyRUEGrowthModel`](@ref), as it also computes the maintenance and growth respiration of the plant, so it has more parameters (we use the default values here). + +We can run a new simulation and see that the simulation's results are different from the previous simulation: + +```@example usepkg +output_updated = run!(models2, meteo_day) +``` + +And that's it! We can switch between models without changing the code, and without having to recompute the dependency graph manually. This is a very powerful feature of PlantSimEngine!💪 + +!!! note + This was a very standard but straightforward example. Sometimes other models will require to add other models to the [`ModelList`](@ref). For example ToyAssimGrowthModel could have required a maintenance respiration model. In this case `PlantSimEngine` will indicate what kind of model is required for the simulation. + +!!! note + In our example we replaced what we call a [soft-dependency coupling](@ref hard_dependency_def), but the same principle applies to [hard-dependencies](@ref hard_dependency_def). Hard and Soft dependencies are concepts related to model coupling, and are discussed in more detail in [Standard model coupling](@ref) and [Coupling more complex models](@ref). + diff --git a/docs/src/step_by_step/parallelization.md b/docs/src/step_by_step/parallelization.md new file mode 100644 index 000000000..2a1d18db9 --- /dev/null +++ b/docs/src/step_by_step/parallelization.md @@ -0,0 +1,38 @@ +## Parallel execution + +!!! note + This page is likely to change and become outdated. In any case, parallel execution only currently applies to single-scale simulations (multi-scale simulations' changing MTGs and extra complexity don't allow for straightforward parallelisation) + +### FLoops + +`PlantSimEngine.jl` uses the [`Floops`](https://juliafolds.github.io/FLoops.jl/stable/) package to run the simulation in sequential, parallel (multi-threaded) or distributed (multi-process) computations over objects, time-steps and independent processes. + +That means that you can provide any compatible executor to the `executor` argument of [`run!`](@ref). By default, [`run!`](@ref) uses the [`ThreadedEx`](https://juliafolds.github.io/FLoops.jl/stable/reference/api/#executor) executor, which is a multi-threaded executor. You can also use the [`SequentialEx`](https://juliafolds.github.io/Transducers.jl/dev/reference/manual/#Transducers.SequentialEx)for sequential execution (non-parallel), or [`DistributedEx`](https://juliafolds.github.io/Transducers.jl/dev/reference/manual/#Transducers.DistributedEx) for distributed computations. + +### Parallel traits + +`PlantSimEngine.jl` uses [Holy traits](https://invenia.github.io/blog/2019/11/06/julialang-features-part-2/) to define if a model can be run in parallel. + +!!! note + A model is executable in parallel over time-steps if it does not uses or set values from other time-steps, and over objects if it does not uses or set values from other objects. + +You can define a model as executable in parallel by defining the traits for time-steps and objects. For example, the ToyLAIModel model from the [examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/tree/main/examples) can be run in parallel over time-steps and objects, so it defines the following traits: + +```julia +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyLAIModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyLAIModel}) = PlantSimEngine.IsObjectIndependent() +``` + +By default all models are considered not executable in parallel, because it is the safest option to avoid bugs that are difficult to catch, so you only need to define these traits if it is executable in parallel for them. + +!!! tip + A model that is defined executable in parallel will not necessarily will. First, the user has to pass a parallel `executor` to [`run!`](@ref) (*e.g.* `ThreadedEx`). Second, if the model is coupled with another model that is not executable in parallel, `PlantSimEngine` will run all models in sequential. + +### Further executors + +You can also take a look at [FoldsThreads.jl](https://github.com/JuliaFolds/FoldsThreads.jl) for extra thread-based executors, [FoldsDagger.jl](https://github.com/JuliaFolds/FoldsDagger.jl) for +Transducers.jl-compatible parallel fold implemented using the Dagger.jl framework, and soon [FoldsCUDA.jl](https://github.com/JuliaFolds/FoldsCUDA.jl) for GPU computations +(see [this issue](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues/22)) and [FoldsKernelAbstractions.jl](https://github.com/JuliaFolds/FoldsKernelAbstractions.jl). You can also take a look at +[ParallelMagics.jl](https://github.com/JuliaFolds/ParallelMagics.jl) to check if automatic parallelization is possible. + +Finally, you can take a look into [Transducers.jl's documentation](https://github.com/JuliaFolds/Transducers.jl) for more information, for example if you don't know what is an executor, you can look into [this explanation](https://juliafolds.github.io/Transducers.jl/stable/explanation/glossary/#glossary-executor). diff --git a/docs/src/step_by_step/quick_and_dirty_examples.md b/docs/src/step_by_step/quick_and_dirty_examples.md new file mode 100644 index 000000000..8aa8d7405 --- /dev/null +++ b/docs/src/step_by_step/quick_and_dirty_examples.md @@ -0,0 +1,95 @@ +# Quick examples + +This page is meant for people who have set up their environment and just want to copy-paste an example or two, see what the REPL returns and start tinkering. + +If you are less comfortable with Julia, or need to set up an environment first, see this page : [Getting started with Julia](@ref). +If you wish for a more detailed rundown of the examples, you can instead have a look at the [step by step](#step_by_step) section, which will go into more detail. + +These examples are all for single-scale simulations. For multi-scale modelling tutorials and examples, refer to [this section][#multiscale] + +You can find the implementation for all the example models, as well as other toy models [in the examples folder](https://github.com/VirtualPlantLab/PlantSimEngine.jl/tree/main/examples). + +```@contents +Pages = ["quick_and_dirty_examples.md"] +Depth = 2 +``` + +## Environment + +These examples assume you have a working Julia environment with PlantSimengine added to it, as well as the other packages used in these examples. Details for getting to that point are provided on the [Installing and running PlantSimEngine](@ref) page. + + +## Example with a single light interception model and a single weather timestep + +```@example usepkg +using PlantSimEngine, PlantMeteo +using PlantSimEngine.Examples +meteo = Atmosphere(T = 20.0, Wind = 1.0, Rh = 0.65, Ri_PAR_f = 500.0) +leaf = ModelList(Beer(0.5), status = (LAI = 2.0,)) +out = run!(leaf, meteo) +``` + +## Coupling the light interception model with a Leaf Area Index model + +The weather data in this example contains data over 365 days, meaning the simulation will have as many timesteps. + +```@example usepkg +using PlantSimEngine +using PlantMeteo, CSV, DataFrames + +using PlantSimEngine.Examples + +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models = ModelList( + ToyLAIModel(), + Beer(0.5), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +outputs_coupled = run!(models, meteo_day) +``` + +## Coupling the light interception and Leaf Area Index models with a biomass increment model + + +```@example usepkg +using PlantSimEngine +using PlantMeteo, CSV, DataFrames + +using PlantSimEngine.Examples + +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +outputs_coupled = run!(models, meteo_day) +``` + +## Example using PlantBioPhysics + +A companion package, PlantBioPhysics, uses PlantSimEngine, and contains other models used in ecophysiological simulations. + +You can have a look at its documentation [here](https://vezy.github.io/PlantBiophysics.jl/stable/) + +Several example simulations are provided there. Here's one taken from [this page](https://vezy.github.io/PlantBiophysics.jl/stable/simulation/first_simulation/) : + +```julia +using PlantBiophysics, PlantSimEngine + +meteo = Atmosphere(T = 22.0, Wind = 0.8333, P = 101.325, Rh = 0.4490995) + +leaf = ModelList( + Monteith(), + Fvcb(), + Medlyn(0.03, 12.0), + status = (Ra_SW_f = 13.747, sky_fraction = 1.0, aPPFD = 1500.0, d = 0.03) + ) + +out = run!(leaf,meteo) +``` \ No newline at end of file diff --git a/docs/src/step_by_step/simple_model_coupling.md b/docs/src/step_by_step/simple_model_coupling.md new file mode 100644 index 000000000..548776f7e --- /dev/null +++ b/docs/src/step_by_step/simple_model_coupling.md @@ -0,0 +1,127 @@ +# Standard model coupling + +```@setup usepkg +using PlantSimEngine +using PlantSimEngine.Examples +using CSV +using DataFrames +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) +nothing +``` + +## Setting up your environment + +Again, make sure you have a working Julia environment with PlantSimengine added to it, and the other recommended companion packages. Details for getting to that point are provided on the [Installing and running PlantSimEngine](@ref) page. + +## ModelList + +The [`ModelList`](@ref) is a container that holds a list of models, their parameter values, and the status of the variables associated to them. + +If one looks at prior examples, the Modellists so far have only contained a single model, whose input variables are initialised in the Modellist [`status`](@ref) keyword argument. + +Example models are all taken from the example scripts in the [`examples`](https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/master/examples/) folder. + +Here's a first [`ModelList`](@ref) declaration with a light interception model, requiring input Leaf Area Index (LAI): + +```julia +modellist_coupling_part_1 = ModelList(Beer(0.5), status = (LAI = 2.0,)) +``` + +Here's a second one with a Leaf Area Index model, with some example Cumulated Thermal Time as input. (This TT_cu is usually computed from weather data): + +```julia +modellist_coupling_part_2 = ModelList( + ToyLAIModel(), + status=(TT_cu=1.0:2000.0,), # Pass the cumulated degree-days as input to the model +) +``` + +## Combining models + +Suppose we want our ToyLAIModel to compute the `LAI` for the light interception model. + +We can couple the two models by having them be part of a single [`ModelList`](@ref). The `LAI` variable will then be a coupled output computed by the ToyLAIModel, then used as input by `Beer`. It will no longer need to be declared as part of the [`status` . + +This is an instance of what we call a ["soft dependency" coupling](@ref hard_dependency_def): a model depends on another model's outputs for its inputs. + +Here's a first attempt : + +```@example usepkg +using PlantSimEngine +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples + +# A ModelList with two coupled models +models = ModelList( + ToyLAIModel(), + Beer(0.5), + status=(TT_cu=1.0:2000.0,), +) +struct UnexpectedSuccess <: Exception end #hack to enable checking an error without failing docbuild #hide +# see https://github.com/JuliaDocs/Documenter.jl/issues/1420 #hide +try #hide +run!(models) +throw(UnexpectedSuccess()) #hide +catch err; err isa UnexpectedSuccess ? rethrow(err) : showerror(stderr, err); end #hide +``` + +Oops, we get an error related to the weather data, with the detailed output being: + +```julia +ERROR: type NamedTuple has no field Ri_PAR_f +Stacktrace: + [1] getindex(mnt::Atmosphere{(), Tuple{}}, i::Symbol) + @ PlantMeteo ~/Path/to/PlantMeteo/src/structs/atmosphere.jl:147 + [2] getcolumn(row::PlantMeteo.TimeStepRow{Atmosphere{(), Tuple{}}}, nm::Symbol) + @ PlantMeteo ~/Path/to/PlantMeteo/src/structs/TimeStepTable.jl:205 + ... +``` + +The `Beer` model requires a specific meteorological parameter. Let's fix that by importing the example weather data : + +```@example usepkg +using PlantSimEngine + +# PlantMeteo and CSV packages are now used +using PlantMeteo, CSV + +# Import the examples defined in the `Examples` sub-module: +using PlantSimEngine.Examples + +# Import example weather data +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +# A ModelList with two coupled models +models = ModelList( + ToyLAIModel(), + Beer(0.5), + status=(TT_cu=cumsum(meteo_day.TT),), # We can now compute a genuine cumulative thermal time from the weather data +) + +# Add the weather data to the run! call +outputs_coupled = run!(models, meteo_day) + +``` + +And there you have it. The light interception model made its computations using the Leaf Area Index computed by ToyLAIModel. + +## Further coupling + +Of course, one can keep adding models. Here's an example ModelList with another model, ToyRUEGrowthModel, which computes the carbon biomass increment caused by photosynthesis. + +```julia +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +nothing # hide +``` \ No newline at end of file diff --git a/docs/src/troubleshooting_and_testing/downstream_tests.md b/docs/src/troubleshooting_and_testing/downstream_tests.md new file mode 100644 index 000000000..92aaf41af --- /dev/null +++ b/docs/src/troubleshooting_and_testing/downstream_tests.md @@ -0,0 +1,9 @@ +# Automated tests : downstream dependency checking + +PlantSimEngine is [open sourced on Github](https://github.com/VirtualPlantLab/PlantSimEngine.jl), and so are its other companion packages, [PlantGeom.jl](https://github.com/VEZY/PlantGeom.jl), [PlantMeteo.jl](https://github.com/VEZY/PlantMeteo.jl), [PlantBioPhysics.jl](https://github.com/VEZY/PlantBioPhysics.jl), [MultiScaleTreeGraph.jl](https://github.com/VEZY/MultiScaleTreeGraph.jl), and [XPalm](https://github.com/PalmStudio/XPalm.jl). + +One handy CI (Continuous Integration) feature implemented for these packages is automated integration and downstream testing: after changes to a package, its known downstream dependencies are tested to ensure no breaking changes were introduced. + +For instance, PlantBioPhysics uses PlantSimEngine, so an integration test ensures that PlantBioPhysics's tests don't break in an unforeseen manner after a new PlantSimEngine release. There also is a benchmark check in the downstream tests: [https://github.com/VirtualPlantLab/PlantSimEngine.jl/blob/main/test/downstream/test-plantbiophysics.jl] + +This is something you can take advantage of if you wish to develop using PlantSimEngine, by providing us with your package name (or adding it to the CI yml file in a Pull Request); we can then add it to the list of downstream packages to test, and generate PR when breaking changes are introduced. \ No newline at end of file diff --git a/docs/src/troubleshooting_and_testing/implicit_contracts.md b/docs/src/troubleshooting_and_testing/implicit_contracts.md new file mode 100644 index 000000000..0197d6a1e --- /dev/null +++ b/docs/src/troubleshooting_and_testing/implicit_contracts.md @@ -0,0 +1,75 @@ +This page summarizes some of the assumptions, coupling constraints and inner workings of PlantSimEngine which may be particular relevant when implementing new models. + +If you are unsure of an implementation subtlety, check this page out to see whether it answers your question. + +```@contents +Pages = ["implicit_contracts.md"] +Depth = 2 +``` + +## Weather data provides the simulation timestep, but models can veer away from it + +The weather data timesteps, whether hourly or daily, provide the pace at which most other models run. + +In XPalm, weather data for most models is provided daily, meaning biomass calculations are also provided daily. + +Many models are considered to be steady-state over that timeframe, but not all : the leaf pruning model pertubes the plant in a non-steady state fashion, for example. Models that require computations over several iterations to stabilise (often part of hard dependencies) might also have a timestep unrelated to the weather data. + +!!! Note + Implicitely, this means any vector variables given as input to the simulation must be consistent with the number of weather timesteps. Providing one weather value but a larger vector variable is an exception : the weather data is replicated over each timestep. (This may be subject to change in the future when support for different timesteps in a single simulation is implemented) + +## Weather data must be interpolated prior to simulation + +If your weather data isn't adjusted to conform to a regular timestep, you will need to adjust it to fit that constraint. PlantSimEngine does no interpolation prior to simulation and expects regular weather timesteps. + +## No cyclic dependencies in the simplified dependency graph + +The model dependency graph used for running the simulation is comprised of soft and hard dependency nodes, and the final version only links soft dependency nodes together, and is expected to contain no cycles. + +Any user model coupling which causes a cyclic dependency to occur will require some extra tinkering to run : either design models differently, create a hard dependency with some of the problematic models, or break the cycle by having a variable take the previous timestep's value as input. + +See [Dependency graphs](@ref) and the following subsections for more discussion related to dependency graph constraints. + +Note : Only the previous timestep is accessible in PlantSimEngine without any kind of dedicated model. How to create a model to store more past timesteps of a specific variable is described in the [Tips and workarounds](@ref) page: [Making use of past states in multi-scale simulations](@ref) + +## Hard dependencies need to be declared in the model definition + +Hard dependencies are handled internally by their owning soft dependency model, ie the hard dep's run! function is directly called by the soft dependency's run!. + +The current way in which PlantSimEngine creates its dependency graph requires users to declare what process is required in the hard dependency and which scale it pulls the model and its variables from. + +## Parallelisation opportunities must be part of the model definition + +Traits that indicate that a model is independent or objects need to be part of the model definition. Modelers need to keep this in mind when implementing new models. + +This is currently mostly a concern for single-scale simulations, as multi-scale simulations are not currently parallelised ; a more involved scheduler would need to be implemented when MTGs are modified by models, and to handle more interesting parallelisation opportunities at specific scales. + +There may be new parallelisation features for multi-plant simulations further down the road. + +## Hard dependencies can only have one parent in the dependency graph + +The final dependency graph is comprised only of soft dependency nodes, and is guaranteed to contain no cycles. Hard dependencies are handled internally by their soft dependency ancestor. To avoid any ambiguity in terms of processing order, only one soft dependency node can 'own' a hard dependency And similarly, nested hard dependencies only have a single soft dependency ancestor. + +This is not solely an implementation detail of PlantSimEngine's internal mechanisms ; if your simulation requires complex coupling, you might need to carefully consider how to manage your hard dependencies, or insert an extra intermediate model to simplify things. + +## A model can only be used once per scale + +Similarly, to avoid depedency graph ambiguity (and for simulation cohesion), PlantSimEngine currently assumes a model describing a process only occurs once per scale. + +Model renaming and duplicating works around this assumption. It may change once multi-plant/multi-species features are implemented. + +## No two variables with the same name at the same scale + +This rule avoids potential ambiguity which could then cause both problems in terms of model ordering during the simulation, as well as incorrectly coupling models with the wrong variable. + +A workaround for some of the situations where this occurs is described here : [Having a variable simultaneously as input and output of a model](@ref) + +## Simulation order instability when adding models + +An important aspect to bear in mind is that PlantSimEngine automatically determines an order in which models are run from the dependency graph it generates by coupling models together. + +This order of simulation depends on the way the models link together. If you replace a model by a new set of models, or pass in new variables that create new links between models, you may change the simulation order. + +When iterating and slowly making a simulation more physiologically realistic and complex, it is therefore fully possible that the order in which two models are run is flipped by a user change. + +This design choice implementation -a concession made for ease of use and flexibility when developing a simulation- means that until your set of models is fully stabilized and you know which variables are `PreviousTimestep` and what order models run in, as you expand and change the set you might see differences of execution of one timestep for some models. It isn't a conceptual problem as most models are steady-state, and simulation order is stable for a given set of models, but it does mean PlantSimEngine will be less conveient for some types of simulation. \ No newline at end of file diff --git a/docs/src/troubleshooting_and_testing/plantsimengine_and_julia_troubleshooting.md b/docs/src/troubleshooting_and_testing/plantsimengine_and_julia_troubleshooting.md new file mode 100644 index 000000000..06fca8d05 --- /dev/null +++ b/docs/src/troubleshooting_and_testing/plantsimengine_and_julia_troubleshooting.md @@ -0,0 +1,496 @@ +# Troubleshooting error messages + +PlantSimEngine attempts to be as comfortable and easy to use as possible for the user, and many kinds of user error will be caught and explanations provided to resolve them, but there are still blind spots, as well as syntax errors that will often generate a Julia error (which can be less intuitive to decrypt) rather than a PlantSimEngine error. + +To help people newer to Julia with troubleshooting, here are a few common 'easy-to-make' mistakes with the current API that might not be obvious to interpret, and pointers on how to fix them. + +They are listed by 'nature of error', rather than by error message, so you may need to search the page to find your specific error. + +If you need more help to decode Julia errors, you can find help on the [Julia Discourse forums](https://discourse.julialang.org). +If you need some advice on the FSPM side, the research community has [its own discourse forum](https://fspm.discourse.group). + +If the issue seems PlantSimEngine-related, or you have questions regarding modeling or have suggestions, you can also [file an issue](https://github.com/VirtualPlantLab/PlantSimEngine.jl/issues) on Github. + +```@contents +Pages = ["plantsimengine_and_julia_troubleshooting.md"] +Depth = 3 +``` + +## Tips and workflow + +Some errors are very specific as to their cause, and the PlantSimEngine errors tend to be explicit about which parameter / variable / organ is causing the error, helping narrow down its origin. + +Some generic-looking errors usually do contain some extra information to help focus the debugging hunt. For instance, a dispatch failure on run! caused by some issue with args/kwargs may highlight explicitely indicate which arguments are currently causing conflict. In VSCode, such arguments are highlighted in red (the first and last arguments in the example below) : + +```julia +a = 1 +run!(a, simple_mtg, mapping, meteo_day, a) + +ERROR: MethodError: no method matching run!(::Int64, ::Node{NodeMTG, Dict{…}}, ::Dict{String, Tuple{…}}, ::DataFrame, ::Int64) +The function [`run!`](@ref) exists, but no method is defined for this combination of argument types. + +Closest candidates are: + run!(::ToyPlantLeafSurfaceModel, ::Any, ::Any, ::Any, ::Any, ::Any) + @ PlantSimEngine /PlantSimEngine/examples/ToyLeafSurfaceModel.jl:75 + ... +``` + +If you wish to search for a specific error in the current page, copy the part of the description that is not specific to your script, and Ctrl+F it here. In the above example, the generic part would be : +```julia +ERROR: MethodError: no method matching +``` + +## Common Julia errors + +### NamedTuples with a single value require a comma : + +This one is easy to miss. + +Empty NamedTuple objects are initialised with x = NamedTuple(). Ones with more than one variable can be initialised like this : +```julia +a = (var1 = 0, var2 = 0) +``` +or like this : +```julia +a = (var1 = 0, var2 = 0,) +``` +The second comma being optional. + +However, if there is only a single variable, notation has to be : +```julia +a = (var1 = 0,) +``` +The comma is compulsory. If it is forgotten : +```julia +a = (var1 = 0) +``` +the line will be interpreted as setting the variable a to the value var1 is set to, hence a will be an Int64 of value 0. + +This is a liability when writing custom models as some functions work with NamedTuples : +```julia +function PlantSimEngine.inputs_(::HardDepSameScaleAvalModel) + (e2 = -Inf,) +end +``` + +The error returned will likely be a Julia error along the lines of : +```julia +[ERROR: MethodError: no method matching merge(::Float64, ::@NamedTuple{g::Float64}) + +Closest candidates are: +merge(::NamedTuple{()}, ::NamedTuple) +@ Base namedtuple.jl:337 +merge(::NamedTuple{an}, ::NamedTuple{bn}) where {an, bn} +@ Base namedtuple.jl:324 +merge(::NamedTuple, ::NamedTuple, NamedTuple...) +@ Base namedtuple.jl:343 + +Stacktrace: +[1] variables_multiscale(node::PlantSimEngine.HardDependencyNode{…}, organ::String, vars_mapping::Dict{…}, st::@NamedTuple{}) +... +``` +It is sometimes properly detected and explained on PlantSimEngine's side (when passing in tracked_outputs, for instance), but may also occur when declaring statuses. + +### Incorrectly declaring empty inputs or outputs + +The syntax for an empty NamedTuple is `NamedTuple()`. If instead one types `()` or `(,)`an error returned respectively by PlantSimEngine or Julia will be returned. + +## PlantSimEngine user errors + +Most of the following errors occur exclusively in multi-scale simulations, which has a slightly more complex API, but some are common to both single- and multi-scale simulations. + +### Implementing a model: forgetting to import or prefix functions + +When implementing a model, you need to make sure that your implementation is correctly recognised as extending `PlantSimEngine` methods and types, and not writing new independent ones. + +In the following working toy model implementation, note that the `inputs_`, `outputs_` and [`run!`](@ref) function are all prefixed with the module name. If there were hard dependencies to manage, the [`dep`](@ref) function would also be identically prefixed. + +```julia +using PlantSimEngine +@process "toy" verbose = false + +struct ToyToyModel{T} <: AbstractToyModel + internal_constant::T +end + +function PlantSimEngine.inputs_(::ToyToyModel) + (a = -Inf, b = -Inf, c = -Inf) +end + +function PlantSimEngine.outputs_(::ToyToyModel) + (d = -Inf, e = -Inf) +end + + +function PlantSimEngine.run!(m::ToyToyModel, models, status, meteo, constants=nothing, extra_args=nothing) + status.d = m.internal_constant * status.a + status.e += m.internal_constant +end + +meteo = Weather([ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=200.0), + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=200.0), + Atmosphere(T=18.0, Wind=1.0, Rh=0.65, Ri_PAR_f=100.0), +]) + +model = ModelList( + ToyToyModel(1), + status = ( a = 1, b = 0, c = 0), +) +to_initialize(model) +sim = PlantSimEngine.run!(model, meteo) +``` + +If you declare these functions without importing them first, or prefixing them with the module name, they will be considered to be part of your current environment, and won't be extending PlantSimEngine methods, which means PlantSimEngine will not be able to properly make use of your functions, and simulations are likely to error, or run incorrectly. + +Forgetting to prefix the [`run!`](@ref) function definition gives the following error : +```julia +ERROR: MethodError: no method matching run!(::ModelList{@NamedTuple{…}, Status{…}}, ::TimeStepTable{Atmosphere{…}}) +The function [`run!`](@ref) exists, but no method is defined for this combination of argument types. + +Closest candidates are: + run!(::ToyToyModel, ::Any, ::Any, ::Any, ::Any, ::Any) + @ Main ~/path/to/file.jl:20 +``` + +Forgetting to prefix the `inputs_`or `outputs_` functions for your model might not always generate an error, depending on whether the variables declared in this function are present in your ModelList or mapping's corresponding Status. + +In cases where they do throw an error, you may get the following kind of output: +```julia +ERROR: type NamedTuple has no field d +Stacktrace: + [1] setproperty!(mnt::Status{(:a, :b, :c), Tuple{…}}, s::Symbol, x::Int64) + @ PlantSimEngine ~/path/to/package/PlantSimEngine/src/component_models/Status.jl:100 + [2] run!(m::ToyToyModel{…}, models::@NamedTuple{…}, status::Status{…}, meteo::PlantMeteo.TimeStepRow{…}, constants::Constants{…}, extra_args::Nothing) + ... +``` + +!!! note + There may be more we can do on our end in the future to make the issue more obvious, but in the meantime it is safest to consistently prefix the methods you need to declare and call with `PlantSimEngine.`, or to explicitely import the functions you wish to extend, *e.g.*: `import PlantSimEngine: inputs_, outputs_`. + +### MultiScaleModel : forgetting a kwarg in the declaration + +A MultiScaleModel requires two kwargs, model and mapped_variables : + +```julia +models = MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[:TT_cu => "Scene",], + ) +``` + +Forgetting 'model=' : + +```julia +models = MultiScaleModel( + ToyLAIModel(), + mapped_variables=[:TT_cu => "Scene",], + ) +ERROR: MethodError: no method matching MultiScaleModel(::ToyLAIModel; mapped_variables::Vector{Pair{Symbol, String}}) +The type `MultiScaleModel` exists, but no method is defined for this combination of argument types when trying to construct it. + +Closest candidates are: + MultiScaleModel(::T, ::Any) where T<:AbstractModel got unsupported keyword argument "mapped_variables" + @ PlantSimEngine PlantSimEngine/src/mtg/MultiScaleModel.jl:188 + MultiScaleModel(; model, mapped_variables) + @ PlantSimEngine PlantSimEngine/src/mtg/MultiScaleModel.jl:191 +``` + +Forgetting 'mapped_variables=' : +```julia +models = MultiScaleModel( + model=ToyLAIModel(), + [:TT_cu => "Scene",], + ) + +ERROR: MethodError: no method matching MultiScaleModel(::Vector{Pair{Symbol, String}}; model::ToyLAIModel) +The type `MultiScaleModel` exists, but no method is defined for this combination of argument types when trying to construct it. + +Closest candidates are: + MultiScaleModel(; model, mapping) + @ PlantSimEngine PlantSimEngine/src/mtg/MultiScaleModel.jl:191 + MultiScaleModel(::T, ::Any) where T<:AbstractModel got unsupported keyword argument "model" +``` + +The message 'got unsupported keyword argument "model"' can be misleading, as in the error in this case is not that a kwarg is *unsupported*, but rather that a keyword argument is *missing*. + +### MultiScaleModel : variable not defined in Module + +A possible cause for this error is that a variable was declared instead of a symbol in a mapping for a multiscale model : + +```julia +mapping = Dict("Scale" => +MultiScaleModel( + model = ToyModel(), + mapped_variables = [should_be_symbol => "Other_Scale"] # should_be_symbol is a variable, likely not found in the current module +), +... +), +``` + +Here's the correct version : +```julia +mapping = Dict("Scale" => +MultiScaleModel( + model = ToyModel(), + mapped_variables=[:should_be_symbol => "Other_Scale"] # should_be_symbol is now a symbol +), +... +), +``` + +### Kwarg and arg parameter issues when calling run! + +There are, unfortunately, multiple ways of passing in arguments to the run! functions that will confuse dynamic dispatch. Some of it is due to imperfections in type declarations on PlantSimEngine's end and may be improved upon in the future. + +Here are a few examples when modifying the usual multiscale run! call in this working example : + +```julia + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) + var1 = 15.0 + + mapping = Dict( + "Leaf" => ( + Process1Model(1.0), + Process2Model(), + Process3Model(), + Status(var1=var1,) + ) + ) + + outs = Dict( + "Leaf" => (:var1,), # :non_existing_variable is not computed by any model + ) + +run!(mtg, mapping, meteo_day, PlantMeteo.Constants(), tracked_outputs=outs) +``` + +The exact signature is this : +```julia +function run!( + object::MultiScaleTreeGraph.Node, + mapping::Dict{String,T} where {T}, + meteo=nothing, + constants=PlantMeteo.Constants(), + extra=nothing; + nsteps=nothing, + tracked_outputs=nothing, + check=true, + executor=ThreadedEx() +``` + +Arguments after the mtg and mapping all have a default value and are optional, and arguments after the ';' delimiter are kwargs and need to be named. + +If one forgets the mtg, a flaw in the way run! is defined will lead to this error : +```julia +run!(mapping, meteo_day, PlantMeteo.Constants(), tracked_outputs=outs) + +ERROR: MethodError: no method matching check_dimensions(::PlantSimEngine.TableAlike, ::Tuple{…}, ::DataFrame) +The function `check_dimensions` exists, but no method is defined for this combination of argument types. + +Closest candidates are: + check_dimensions(::Any, ::Any) + @ PlantSimEngine PlantSimEngine/src/checks/dimensions.jl:43 + ... +``` + +If one forgets the necessary 'tracked_outputs=' in the definition, outs will be interpreted as the 'extra' arg instead of a kwarg. 'extra' usually defaults to nothing, and is reserved in multiscale mode, leading to the following error : + +```julia +run!(mtg, mapping, meteo_day, PlantMeteo.Constants(), outs) + +ERROR: Extra parameters are not allowed for the simulation of an MTG (already used for statuses). +Stacktrace: + [1] error(s::String) + @ Base ./error.jl:35 + [2] run!(::PlantSimEngine.TreeAlike, object::PlantSimEngine.GraphSimulation{…}, meteo::DataFrames.DataFrameRows{…}, constants::Constants{…}, extra::Dict{…}; tracked_outputs::Nothing, check::Bool, executor::ThreadedEx{…}) +``` + +In case of a more generic error that returns a +For example, if one does the opposite and adds a non-existent kwarg, the generic dispatch failure has some more specific information : +`got unsupported keyword argument "constants"` + +```julia +run!(mtg, mapping, meteo_day, constants=PlantMeteo.Constants(), tracked_outputs=outs) + +ERROR: MethodError: no method matching run!(::Node{…}, ::Dict{…}, ::DataFrame, ::Dict{…}, ::Nothing; constants::Constants{…}) +This error has been manually thrown, explicitly, so the method may exist but be intentionally marked as unimplemented. + +Closest candidates are: + run!(::Node, ::Dict{String}, ::Any, ::Any, ::Any; nsteps, tracked_outputs, check, executor) got unsupported keyword argument "constants" +``` + +### Hard dependency process not present in the mapping + +Another weakness in the current error checking leads to an unclear Julia error if a model A is present in a mapping and has a hard dependency on a model B, but B is absent from the mapping. + +In the following example, A corresponds to Process3Model, which requires a model B implementing 'Process2Model' and referred to as 'process2'. +Looking at the source code for Process3Model, the hard dependency is declared here : +```julia +PlantSimEngine.dep(::Process3Model) = (process2=Process2Model,) +``` + +However, the model provided in the examples, Process2Model is absent from the mapping : + +```julia +simple_mtg = Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 1)) +mapping = Dict( + "Leaf" => ( + Process3Model(), + Status(var5=15.0,) + ) +) +outs = Dict( + "Leaf" => (:var5,), +) +run!(simple_mtg, mapping, meteo_day, tracked_outputs=outs) + +ERROR: type NamedTuple has no field process2 +Stacktrace: + [1] getproperty(x::@NamedTuple{process3::Process3Model}, f::Symbol) + @ Base ./Base.jl:49 + [2] run!(::Process3Model, models::@NamedTuple{…}, status::Status{…}, meteo::DataFrameRow{…}, constants::Constants{…}, extra::PlantSimEngine.GraphSimulation{…}) + ... +``` + +The fix is to add Process2Model() -or another model for the same process- to the mapping. + +### Status API ambiguity + +One current problem with PlantSimEngine's API is that declaring a simulation's Status or Statuses differs between single- and multi-scale. + +Returning to the example in [Implementing a model: forgetting to import or prefix functions](@ref), the `ModelList` status was declared like this: + +```julia +model = ModelList( + ToyToyModel(1), + status = ( a = 1, b = 0, c = 0), +) +``` +If instead you replace `status = ...`with the multi-scale declaration: `Status(...)`, you will get the following error: + +```julia +ERROR: MethodError: no method matching process(::Status{(:a, :b, :c), Tuple{Base.RefValue{Int64}, Base.RefValue{Int64}, Base.RefValue{Int64}}}) +The function `process` exists, but no method is defined for this combination of argument types. + +Closest candidates are: + process(::Pair{Symbol, A}) where A<:AbstractModel + @ PlantSimEngine ~/path/to/pkg/PlantSimEngine/src/Abstract_model_structs.jl:16 + process(::A) where A<:AbstractModel + @ PlantSimEngine ~/path/to/pkg/PlantSimEngine/src/Abstract_model_structs.jl:13 + +Stacktrace: + [1] (::PlantSimEngine.var"#5#6")(i::Status{(:a, :b, :c), Tuple{Base.RefValue{…}, Base.RefValue{…}, Base.RefValue{…}}}) + @ PlantSimEngine ./none:0 + [2] iterate +``` + +If you do the opposite in a multi-scale simulation by replacing the necessary `Status(...)` with `status = ...`, you may get an `ERROR: syntax: invalid named tuple element` error. Here's some output when tinkering with the Toy Plant tutorial's mapping: + +```julia +ERROR: syntax: invalid named tuple element "MultiScaleModel(...)" around /path/to/Pkg/PlantSimEngine/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl:196 +Stacktrace: + [1] top-level scope + @ ~/path/to/pkg/PlantSimEngine/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl:196 +``` +or +```julia +ERROR: syntax: invalid named tuple element "ToyRootGrowthModel(50, 10)" around /path/to/Pkg/PlantSimEngine/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl:196 +Stacktrace: + [1] top-level scope + @ ~/path/to/Pkg/PlantSimEngine/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl:196 +``` + +## Forgetting to declare a scale in the mapping but having variables point to it + +If there is a need to collect variables at two different scales, and one scale is completely absent from the mapping, the error currently occurs on the Julia side : + +```julia +# No models at the E3 scale in the mapping ! + +"E2" => ( + MultiScaleModel( + model = HardDepSameScaleEchelle2Model(), + mapped_variables=[:c => "E1" => :c, :e3 => "E3" => :e3, :f3 => "E3" => :f3,], + ), + ), + +Exception has occurred: KeyError +* +KeyError: key "E3" not found +Stacktrace: +[1] hard_dependencies(mapping::Dict{String, Tuple{Any, Any}}; verbose::Bool) +@ PlantSimEngine ......./src/dependencies/hard_dependencies.jl:175 +... +``` + +### Parenthesis placement when declaring a mapping + +An unintuitive error encountered in the past when defining a mapping : + +```julia +ERROR: ArgumentError: AbstractDict(kv): kv needs to be an iterator of 2-tuples or pairs +``` +may occur when forgetting the parenthesis after '=>' in a mapping declaration, and combining it with another parenthesis error. +```julia +mapping = Dict( "Scale" => (ToyAssimGrowthModel(0.0, 0.0, 0.0), ToyCAllocationModel(), Status( TT_cu=Vector(cumsum(meteo_day.TT))), ), ) +``` + +Other errors such as : +```julia +ERROR: MethodError: no method matching Dict(::Pair{String, ToyAssimGrowthModel{Float64}}, ::ToyCAllocationModel, ::Status{(:TT_cu,), Tuple{Base.RefValue{…}}}) +The type `Dict` exists, but no method is defined for this combination of argument types when trying to construct it. + +Closest candidates are: + Dict(::Pair{K, V}...) where {K, V} +``` +often indicate a likely syntax error somewhere in the mapping definition. + +### Empty status vectors in multi-scale simulations + +This situation won't trigger an error. Unexpectedly empty vectors can be returned as outputs if you happen to forget to a node at the corresponding scale in the MTG, and no organ creation occurs for that node. + +Here's an example taken from the [Converting a single-scale simulation to multi-scale](@ref) page. It was modified by removing the "Plant" node in the dummy MTG passed into the [`run!`](@ref)function. Without that "Plant" node, only "Scene"-scale models can run initially, and since no nodes are created, "Plant"-scale models will never be run. + +```julia +PlantSimEngine.@process "tt_cu" verbose = false + +struct ToyTt_CuModel <: AbstractTt_CuModel end + +function PlantSimEngine.run!(::ToyTt_CuModel, models, status, meteo, constants, extra=nothing) + status.TT_cu += + meteo.TT +end + +function PlantSimEngine.inputs_(::ToyTt_CuModel) + NamedTuple() # No input variables +end + +function PlantSimEngine.outputs_(::ToyTt_CuModel) + (TT_cu=-Inf,) +end + +mapping_multiscale = Dict( + "Scene" => ToyTt_CuModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.5), + ToyRUEGrowthModel(0.2), + ), +) + +mtg_multiscale = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 0, 0),) +#plant = MultiScaleTreeGraph.Node(mtg_multiscale, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + +out_multiscale = run!(mtg_multiscale, mapping_multiscale, meteo_day) + +out_multiscale["Plant"][:LAI] +``` + +In the above code, uncommenting the second line will add a "Plant" node to the MTG, and the simulation will then behave as intuitively expected. \ No newline at end of file diff --git a/docs/src/model_coupling/tips_and_workarounds.md b/docs/src/troubleshooting_and_testing/tips_and_workarounds.md similarity index 56% rename from docs/src/model_coupling/tips_and_workarounds.md rename to docs/src/troubleshooting_and_testing/tips_and_workarounds.md index 8b156d6b9..07769d715 100644 --- a/docs/src/model_coupling/tips_and_workarounds.md +++ b/docs/src/troubleshooting_and_testing/tips_and_workarounds.md @@ -10,13 +10,18 @@ There are also a couple of features that are quick hacks or that are meant for q We'll list a few of them here, and will likely add some entry in the future listing some built-in limitations or implicit expectations of the package. +```@contents +Pages = ["tips_and_workarounds.md"] +Depth = 2 +``` + ## Making use of past states in multi-scale simulations -It is possible to make use of the value of a variable in the past simulation timestep via the `PreviousTimeStep` mechanism in the mapping API (In fact, as mentioned elsewhere, it is the default way to break undesirable cyclic dependencies that can come up when coupling models, see : [Avoiding cyclic dependencies](@ref)). +It is possible to make use of the value of a variable in the past simulation timestep via the [`PreviousTimeStep`](@ref) mechanism in the mapping API (In fact, as mentioned elsewhere, it is the default way to break undesirable cyclic dependencies that can come up when coupling models, see : [Avoiding cyclic dependencies](@ref)). However, it is not possible to go beyond that through the mapping API. Something like `PreviousTimeStep(PreviousTimeStep(PreviousTimeStep(:carbon_biomass)))` is not supported. Don't do that. -One way to access prior variable states is simply to write an ad hoc model that stores a few values into an array or however many variables you might need, which you can then feed into other models that might need it. +One way to access prior variable states is simply to write an ad hoc model that stores a few values into an array or however many variables you might need, which you can then update every timestep and feed into other models that might need it. ## Having a variable simultaneously as input and output of a model @@ -26,13 +31,26 @@ One current limitation of `PlantSimEngine` that can be occasionally awkward is t The reason being that it is usually impossible to automatically determine how the coupling is supposed to work out, when other dependencies latch onto such a model. The user would have to explicitely declare some order of simulation between several models, and some amount of programmer work would also be necessary to implement that extra API feature into `PlantSimEngine`. -We haven't found an approach that was fully satisfactory from both a code simplicity and an API convenience POV. Especially when prototyping and adding in new models, as that might require redeclaring the simulation order for those specific variables. Ideas that came to mind felt like they might constrain the codebase for a more complex API, without enough benefit to justify it. +We haven't found an approach that was fully satisfactory from both a code simplicity and an API convenience POV. Especially when prototyping and adding in new models, as that might require redeclaring the simulation order for those specific variables. + +There are two workarounds : + +- One possibly awkward approach is to rename one of the variables. It is not ideal, of course, as it means you might not be able to use a predefined model 'out of the box', but it does not have any of the tradeoffs and constraints mentioned above. + +- In many other situations one can work with what PlantSimEngine already provides. + +For example, one model in [XPalm.jl](https://github.com/PalmStudio/XPalm.jl/blob/main/src/plant/phytomer/leaves/leaf_pruning.jl) handles leaf pruning, affecting biomass. A straightforward implementation would be to have a `leaf_biomass` variable as both input and output. The workaround is to instead output a variable `leaf_biomass_pruning_loss` and to have that as input in the next timestep to compute the new leaf biomass. -The current workaround, while a little annoying, is very simple : *rename one of the variables*. It is not ideal, of course, as it means you might not be able to use a predefined model 'out of the box', but it does not have any of the tradeoffs and constrained mentioned above. +[Part 3](../multiscale/multiscale_example_3.md) of the Toy Plant tutorial does something similar for its carbon stock. The `carbon_stock` variable indicates how much carbon is available for root and internode growth, but instead of updating it and passing it along after the root growth decision model decided whether or not roots should be added, that model computes a `carbon_stock_updated_after_roots` which is then used by the internode growth model. -## Passing in a vector in a mapping status at a specific scale +This change in design avoids model order ambiguity and also improves readability, and makes sense in terms of PlantSimEngine's philosophy. -You may have noticed that sometimes a vector (1-dimensional array) variable is passed into the `status` component of a `ModelList` in documentation examples (An example here with cumulative thermal time : [Model switching](@ref)). +## [Multiscale : passing in a vector in a mapping status at a specific scale](@id multiscale_vector) + +!!! note + This section is a little more advanced and not recommended for beginners + +You may have noticed that sometimes a vector (1-dimensional array) variable is passed into the [`status`](@ref) component of a [`ModelList`](@ref) in documentation examples (An example here with cumulative thermal time : [Model switching](@ref)). This is practical for simple simulations, or when quickly prototyping, to avoid having to write a model specifically for it. Whatever models make use of that variable are provided with one element corresponding to the current timestep every iteration. @@ -44,9 +62,9 @@ Due to, uh, implementation quirks, the way to use this is as follows : Call the function `replace_mapping_status_vectors_with_generated_models(mapping_with_vectors_in_status, timestep_model_organ_level, nsteps)`on your mapping. -It will parse your mapping, generate custom models to store and feed the vector values each timestep, and return the new mapping you can then use for your simulation. It also slips in a couple of internal models that provide the timestep index to these models (so note that symbols `:current_timestep` and `:next_timestep` will be declared for that mapping). You can decide which scale/organ level you want those models to be in via the `timestep_model_organ_level`parameter. `nsteps`is used as a sanity check, and expects you to provide the amount of simulation timesteps. +It will parse your mapping, generate custom models to store and feed the vector values each timestep, and return the new mapping you can then use for your simulation. It also slips in a couple of internal models that provide the timestep index to these models (so note that symbols `:current_timestep` and `:next_timestep` will be declared for that mapping). You can decide which scale/organ level you want those models to be in via the `timestep_model_organ_level`parameter. `nsteps` is used as a sanity check, and expects you to provide the amount of simulation timesteps. -!!! note +!!! warning Only subtypes of AbstractVector present in statuses will be affected. In some cases, meteo values might need a small conversion. For instance : ``` meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) @@ -55,4 +73,40 @@ It will parse your mapping, generate custom models to store and feed the vector cumsum(meteo_day.TT) actually returns a CSV.SentinelArray.ChainedVectors{T, Vector{T}}, which is not a subtype of AbstractVector. Replacing it with Vector(cumsum(meteo_day.TT)) will provide an adequate type. -This feature is likely to break in simulations that make use of planned future features (such as mixing models with different timesteps), without guarantee of a fix on a short notice. Again, bear in mind it is mostly a convenient shortcut for prototyping, when doing multi-scale simulations. \ No newline at end of file +Here's an example usage, fixing the first attempt at [Converting a single-scale simulation to multi-scale +](@ref): + + +```julia +using PlantSimEngine +using PlantSimEngine.Examples +using PlantMeteo, CSV, DataFrames +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +# Direct translation of the single-scale simulation +mapping_pseudo_multiscale = Dict( +"Plant" => ( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + Status(TT_cu=cumsum(meteo_day.TT),) + ), +) + +mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 0),) + +# will generate an error as vectors can't be directly passed into a Status in multi-scale simulations +#out_pseudo_multiscale_error = run!(mtg, mapping_pseudo_multiscale, meteo_day) + +mapping_pseudo_multiscale_adjusted = PlantSimEngine.replace_mapping_status_vectors_with_generated_models(mapping_pseudo_multiscale, "Plant", PlantSimEngine.get_nsteps(meteo_day)) + +out_pseudo_multiscale_successful = run!(mtg, mapping_pseudo_multiscale_adjusted, meteo_day) + +``` + + +This feature is likely to break in simulations that make use of planned future features (such as mixing models with different timesteps), without guarantee of a fix on a short notice. Again, bear in mind it is mostly a convenient shortcut for prototyping, when doing multi-scale simulations. + +## Cyclic dependencies in single-scale simulations + +Cyclic dependencies can happen in single-scale simulations, but the PreviousTimestep feature currently isn't available. Hard dependencies are one way to deal with them, creating a multi-scale simulation with a single effective scale is also an option. \ No newline at end of file diff --git a/docs/src/fitting.md b/docs/src/working_with_data/fitting.md similarity index 100% rename from docs/src/fitting.md rename to docs/src/working_with_data/fitting.md diff --git a/docs/src/working_with_data/floating_point_accumulation_error.md b/docs/src/working_with_data/floating_point_accumulation_error.md new file mode 100644 index 000000000..12e09b20a --- /dev/null +++ b/docs/src/working_with_data/floating_point_accumulation_error.md @@ -0,0 +1,163 @@ +# Floating-point considerations + +```@setup usepkg +using PlantSimEngine +using PlantSimEngine.Examples +using PlantMeteo, MultiScaleTreeGraph, CSV +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +out_singlescale = run!(models, meteo_day) +``` +## Investigating a discrepancy + +In the [Converting a single-scale simulation to multi-scale](@ref) page, a single-scale simulation was converted to an equivalent multiscale simulation, and outputs were compared. One detail that was glossed over, but important to bear in mind as a PlantSimEngine user is related to floating-point approximations. + +### Single-scale simulation + +```@example usepkg +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +models_singlescale = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +outputs_singlescale = run!(models_singlescale, meteo_day) +``` + +### Multi-scale equivalent + +```@example usepkg +PlantSimEngine.@process "tt_cu" verbose = false + +struct ToyTt_CuModel <: AbstractTt_CuModel end + +function PlantSimEngine.run!(::ToyTt_CuModel, models, status, meteo, constants, extra=nothing) + status.TT_cu += + meteo.TT +end + +function PlantSimEngine.inputs_(::ToyTt_CuModel) + NamedTuple() # No input variables +end + +function PlantSimEngine.outputs_(::ToyTt_CuModel) + (TT_cu=0.0,) +end + +mapping_multiscale = Dict( + "Scene" => ToyTt_CuModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.5), + ToyRUEGrowthModel(0.2), + ), +) + +mtg_multiscale = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 0, 0),) + plant = MultiScaleTreeGraph.Node(mtg_multiscale, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + +outputs_multiscale = run!(mtg_multiscale, mapping_multiscale, meteo_day) +``` + +### Output comparison + +```@setup usepkg +mapping_multiscale = Dict( + "Scene" => ToyTt_CuModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.5), + ToyRUEGrowthModel(0.2), + ), +) + +mtg_multiscale = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 0, 0),) + plant = MultiScaleTreeGraph.Node(mtg_multiscale, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + +outputs_multiscale = run!(mtg_multiscale, mapping_multiscale, meteo_day) +computed_TT_cu_multiscale = collect(Base.Iterators.flatten(outputs_multiscale["Scene"][:TT_cu])) +``` + +```@example usepkg + +computed_TT_cu_multiscale = collect(Base.Iterators.flatten(outputs_multiscale["Scene"][:TT_cu])) + +is_approx_equal = length(unique(computed_TT_cu_multiscale .≈ outputs_singlescale.TT_cu)) == 1 +``` + +Why was the comparison only approximate ? Why `≈` instead of `==`? + +Let's try it out. What if write instead: + +```@example usepkg +computed_TT_cu_multiscale = collect(Base.Iterators.flatten(outputs_multiscale["Scene"][:TT_cu])) + +is_perfectly_equal = length(unique(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)) == 1 +``` + +Why is this false? Let's look at the data. + +Looking more closely at the output, we can notice that values are identical up to timestep #105 : + +```@example usepkg +(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)[104] +``` + +```@example usepkg +(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)[105] +``` + +We have the values 132.33333333333331 (multi-scale) and 132.33333333333334 (single-scale). The final output values are : 2193.8166666666643 (multi-scale) and 2193.816666666666 (single-scale). + +The divergence isn't huge, but in other situations or over more timesteps it could start becoming a problem. + +## Floating-point summation + +The reason values aren't identical, is due to the fact that many numbers do not have an exact floating point representation. A classical example is the fact that [0.1 + 0.2 != 0.3](https://blog.reverberate.org/2016/02/06/floating-point-demystified-part2.html) : + +```@example usepkg +println(0.1 + 0.2 - 0.3) +``` + +When summing many numbers, depnding on the order in which they are summed, floating-point approximation errors may aggregate more or less quickly. + +The default summation per-timestep in our example `Toy_Tt_CuModel` was a naive summation. The `cumsum` function used in the single-scale simulation to directly compute the TT_cu uses a pairwise summation method that provides approximation error on fewer digits compared to naive summation. Errors aggregate more slowly. + +In our simple example, using Float64 values, the difference wasn't significant enough to matter, but if you are writing a simulation over many timesteps or aggregating a value over many nodes, you may need to alter models to avoid numerical errors blowing up due to floating-point accuracy. + +Depending on what value is being computed and the mathematical operations used, changes may range from applying a simple scale to a range of values, to significant refactoring. + + +## Other links related to floating-point numerical concerns + +Note that many of the examples in these blogposts discuss Float32 accuracy. Float64 values have several extra precision bits to work. + +A series of blog posts on floating-point accuracy: [https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/](https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/) +Floating-Point Visually Explained : [https://fabiensanglard.net/floating_point_visually_explained/](https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/) +Examples of floating point problems: [https://jvns.ca/blog/2023/01/13/examples-of-floating-point-problems/](https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/) + +Relating specifically to floating-point sums: + +Pairwise summation: [https://en.wikipedia.org/wiki/Pairwise_summation](https://en.wikipedia.org/wiki/Pairwise_summation) +Kahan summation: [https://en.wikipedia.org/wiki/Kahan_summation_algorithm](https://en.wikipedia.org/wiki/Kahan_summation_algorithm) +Taming Floating-Point Sums: [https://orlp.net/blog/taming-float-sums/](https://orlp.net/blog/taming-float-sums/) \ No newline at end of file diff --git a/docs/src/extending/inputs.md b/docs/src/working_with_data/inputs.md similarity index 81% rename from docs/src/extending/inputs.md rename to docs/src/working_with_data/inputs.md index 87716770e..4e5b4d004 100644 --- a/docs/src/extending/inputs.md +++ b/docs/src/working_with_data/inputs.md @@ -30,4 +30,15 @@ To do so, you need to implement the following methods for your structure that de - (Optionnally) `PlantMeteo.row_from_parent(row, i)`: return row `i` from the parent table, *e.g.* the row `i` from the DataFrame. This is only needed if you want high performance, the default implementation calls `Tables.rows(parent(row))[i]`. !!! compat - `PlantMeteo.rownumber` is temporary. It soon will be replaced by `DataAPI.rownumber` instead, which will be also used by *e.g.* DataFrames.jl. See [this Pull Request](https://github.com/JuliaData/DataAPI.jl/issues/60). \ No newline at end of file + `PlantMeteo.rownumber` is temporary. It soon will be replaced by `DataAPI.rownumber` instead, which will be also used by *e.g.* DataFrames.jl. See [this Pull Request](https://github.com/JuliaData/DataAPI.jl/issues/60). + +## Working with weather data + +Here's a quick example showcasing how to export the example weather data to your own file : + +```julia +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +PlantMeteo.write_weather("examples/meteo_day.csv", meteo_day, duration = Dates.Day) +``` + +If you wish to filter weather data, reshape it, adjust it, write it, you'll find some more examples in PlantMeteo's [API reference](https://palmstudio.github.io/PlantMeteo.jl/stable/API/). \ No newline at end of file diff --git a/docs/src/reducing_dof.md b/docs/src/working_with_data/reducing_dof.md similarity index 97% rename from docs/src/reducing_dof.md rename to docs/src/working_with_data/reducing_dof.md index 8f6b53aa3..13fe72b20 100644 --- a/docs/src/reducing_dof.md +++ b/docs/src/working_with_data/reducing_dof.md @@ -16,7 +16,7 @@ end ## Introduction -### Why reducing the degrees of freedom +### Why reduce the degrees of freedom Reducing the degrees of freedom in a model, by forcing certain variables to measurements, can be useful for several reasons: @@ -78,16 +78,14 @@ m2 = ModelList( status=(var0 = 0.5, var9 = 10.0), ) -run!(m2, meteo) - -status(m2) +out = run!(m2, meteo) ``` And that's it ! The models that depend on `var9` will now use the measured value of `var9` instead of the one computed by `Process7Model`. ### Hard-coupled models -It is a bit more complicated to reduce the degrees of freedom in a model that is hard-coupled to another model, because it calls the `run!` method of the other model. +It is a bit more complicated to reduce the degrees of freedom in a model that is hard-coupled to another model, because it calls the [`run!`](@ref) method of the other model. In this case, we need to replace the old model with a new model that forces the value of the variable to the measurement. This is done by giving the measurements as inputs of the new model, and returning nothing so the value is unchanged. @@ -116,9 +114,7 @@ m3 = ModelList( status = (var0=0.5,var3 = 10.0) ) -run!(m3, meteo) - -status(m3) +out = run!(m3, meteo) ``` !!! note diff --git a/docs/src/working_with_data/visualising_outputs.md b/docs/src/working_with_data/visualising_outputs.md new file mode 100644 index 000000000..54c7a940f --- /dev/null +++ b/docs/src/working_with_data/visualising_outputs.md @@ -0,0 +1,97 @@ +```@setup usepkg +# ] add PlantSimEngine, DataFrames, CSV +using PlantSimEngine, PlantMeteo, DataFrames, CSV + +# Include the model definition from the examples folder: +using PlantSimEngine.Examples + +# Import the example meteorological data: +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +# Define the list of models for coupling: +model = ModelList( + ToyLAIModel(), + Beer(0.6), + status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model +) + +# Run the simulation: +sim_out = run!(model, meteo_day) + +``` + +# Visualizing outputs and data + +## Output structure + +PlantSimEngine's run! functions return for each timestep the state of the variables that were requested using the `tracked_outputs` kwarg (or the state of every variable if this kwarg was left unspecified). Multi-scale simulations also indicate which organ and MTG node these state variables are related to. + +Here's an example indicating how to plot output data using CairoMakie, a package used for plotting. + +```@example usepkg +# ] add PlantSimEngine, DataFrames, CSV +using PlantSimEngine, PlantMeteo, DataFrames, CSV + +# Include the model definition from the examples folder: +using PlantSimEngine.Examples + +# Import the example meteorological data: +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +# Define the list of models for coupling: +models = ModelList( + ToyLAIModel(), + Beer(0.6), + status=(TT_cu=cumsum(meteo_day[:, :TT]),), # Pass the cumulated degree-days as input to `ToyLAIModel`, this could also be done using another model +) + +# Run the simulation: +sim_outputs = run!(models, meteo_day) +``` + +The output data is displayed as a by default as a `TimeStepTable`. It is also possible to filter which variables are kept via the optional `tracked_outputs` keyword argument. + +## Plotting outputs + +Using CairoMakie, one can plot out selected variables : + +!!! note + You will need to add CairoMakie to your environment through Pkg mode first. + +```@example usepkg +# Plot the results: +using CairoMakie + +fig = Figure(resolution=(800, 600)) +ax = Axis(fig[1, 1], ylabel="LAI (m² m⁻²)") +lines!(ax, sim_outputs[:TT_cu], sim_outputs[:LAI], color=:mediumseagreen) + +ax2 = Axis(fig[2, 1], xlabel="Cumulated growing degree days since sowing (°C)", ylabel="aPPFD (mol m⁻² d⁻¹)") +lines!(ax2, sim_outputs[:TT_cu], sim_outputs[:aPPFD], color=:firebrick1) + +fig +``` + +## TimeStepTables and DataFrames + +```@setup usepkg +sim_out = run!(model, meteo_day) +``` + +The output data is usually stored in a `TimeStepTable` structure defined in `PlantMeteo.jl`, which is a fast DataFrame-like structure with each time step being a [`Status`](@ref). It can be also be any `Tables.jl` structure, such as a regular `DataFrame`. Weather data is also usually stored in a `TimeStepTable` but with each time step being an `Atmosphere`. + +Another simple way to get the results is to transform the outputs into a `DataFrame`. Which is very easy because the `TimeStepTable` implements the Tables.jl interface: + +```@example usepkg +using DataFrames +sim_outputs_df = PlantSimEngine.convert_outputs(sim_outputs, DataFrame) +sim_outputs_df[[1, 2, 3, 363, 364, 365], :] +``` + +It is also possible to create DataFrames from specific variables: + +```julia +df = DataFrame(aPPFD=sim_outputs[:aPPFD][1], LAI=sim_outputs.LAI[1], Ri_PAR_f=meteo.Ri_PAR_f[1]) +``` + +Which can also be useful for [Parameter fitting ](@ref). \ No newline at end of file diff --git a/docs/src/www/GUID-12E2DDAD-7B20-4FE2-AA36-7FAC950382A6-low.png b/docs/src/www/GUID-12E2DDAD-7B20-4FE2-AA36-7FAC950382A6-low.png new file mode 100644 index 000000000..9c3caccc2 Binary files /dev/null and b/docs/src/www/GUID-12E2DDAD-7B20-4FE2-AA36-7FAC950382A6-low.png differ diff --git a/docs/src/www/Grassy_plant_MTG_vertical.svg b/docs/src/www/Grassy_plant_MTG_vertical.svg new file mode 100644 index 000000000..7a7d31eb1 --- /dev/null +++ b/docs/src/www/Grassy_plant_MTG_vertical.svg @@ -0,0 +1,1148 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Diagram + MTG + + + + < + + < + + < + + < + + / + + / + + + + I1 + + I2 + + + + I3 + + + + I4 + + I5 + + I6 + + P + + A1 + + A2 + + + + + + + + / + + diff --git a/docs/src/www/Grassy_plant_scales.svg b/docs/src/www/Grassy_plant_scales.svg new file mode 100644 index 000000000..227ca4a64 --- /dev/null +++ b/docs/src/www/Grassy_plant_scales.svg @@ -0,0 +1,788 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + P + + A1 + + + + + + + A2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Plant + + Axes + (a) + (b) + Phytomer + (c) + + + + < + + < + + + + I1 + + + + I2 + + I3 + + I4 + + < + + I5 + + < + + I6 + + + + diff --git a/docs/src/www/LAI_growth2.png b/docs/src/www/LAI_growth2.png new file mode 100644 index 000000000..8f45a8c7c Binary files /dev/null and b/docs/src/www/LAI_growth2.png differ diff --git a/docs/src/www/MTG_output.png b/docs/src/www/MTG_output.png new file mode 100644 index 000000000..36e660a4e Binary files /dev/null and b/docs/src/www/MTG_output.png differ diff --git a/docs/src/www/PBP_dependency_graph.png b/docs/src/www/PBP_dependency_graph.png new file mode 100644 index 000000000..94aa87866 Binary files /dev/null and b/docs/src/www/PBP_dependency_graph.png differ diff --git a/docs/src/www/Turnstile_state_machine_colored.svg.png b/docs/src/www/Turnstile_state_machine_colored.svg.png new file mode 100644 index 000000000..bd03de112 Binary files /dev/null and b/docs/src/www/Turnstile_state_machine_colored.svg.png differ diff --git a/docs/src/www/dags_acyclic_vs_cyclic-d1a669bf1b8b6bfa8ac3041788e81171.png b/docs/src/www/dags_acyclic_vs_cyclic-d1a669bf1b8b6bfa8ac3041788e81171.png new file mode 100644 index 000000000..eafed015f Binary files /dev/null and b/docs/src/www/dags_acyclic_vs_cyclic-d1a669bf1b8b6bfa8ac3041788e81171.png differ diff --git a/docs/src/www/ecophysio_coupling_diagram.png b/docs/src/www/ecophysio_coupling_diagram.png new file mode 100644 index 000000000..06f431a19 Binary files /dev/null and b/docs/src/www/ecophysio_coupling_diagram.png differ diff --git a/docs/src/www/l1_flux-vs-tensorflow.png b/docs/src/www/l1_flux-vs-tensorflow.png new file mode 100644 index 000000000..85787f4b9 Binary files /dev/null and b/docs/src/www/l1_flux-vs-tensorflow.png differ diff --git a/docs/src/www/mtg_plot_1.svg b/docs/src/www/mtg_plot_1.svg new file mode 100644 index 000000000..481ca3996 --- /dev/null +++ b/docs/src/www/mtg_plot_1.svg @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/src/www/toy_plant.png b/docs/src/www/toy_plant.png new file mode 100644 index 000000000..799294b04 Binary files /dev/null and b/docs/src/www/toy_plant.png differ diff --git a/docs/src/www/toy_plant_stem_only.png b/docs/src/www/toy_plant_stem_only.png new file mode 100644 index 000000000..f7df0c2a9 Binary files /dev/null and b/docs/src/www/toy_plant_stem_only.png differ diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation1.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation1.jl new file mode 100644 index 000000000..f32284b76 --- /dev/null +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation1.jl @@ -0,0 +1,140 @@ + +########################################### +# Toy plant model +# Physiologically meaningless but illustrates organ creation +########################################### + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end + +PlantSimEngine.@process "organ_emergence" verbose = false + +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T +end + +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0, leaves_max_surface_area=100.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area) + +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0, carbon_stock=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) + +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if not enough carbon, no organ creation + if status.carbon_stock < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end + +########################## +### Model accumulating carbon resources +########################## + +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(carbon_captured=0.0,carbon_organ_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (carbon_stock=-Inf,) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsObjectIndependent() + +######################## +## Leaf model capturing some arbitrary carbon quantity +######################## + +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple()#(TT_cu=-Inf) +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + # very crude approximation with LAI of 1 and constant PPFD + status.carbon_captured = 200.0 *(1.0 - exp(-0.2)) +end + +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsTimeStepIndependent() + +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :carbon_organ_creation_consumed=>["Internode"] + ], + ), + Status(carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:carbon_stock)=>"Plant"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) + + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) +#MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "Soil", 1, 1)) + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + + outs = run!(mtg, mapping, meteo_day) + mtg + + length(MultiScaleTreeGraph.traverse(mtg,x->x, symbol="Leaf")) diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl new file mode 100644 index 000000000..a06bd6c2c --- /dev/null +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation2.jl @@ -0,0 +1,237 @@ +########################################### +# Toy plant model +# Physiologically and physically completely meaningless +# (no dimension for units, arbitrary values, stores water and carbon in abstract stocks, +# arbitrary max leaf count and root length, constant and non-coupled photosynthesis and water absorption, ...) +# But it should illustrate the basics of simulating a growing multiscale plant with PlantSimEngine's model approach +########################################### + +function get_root_end_node(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return MultiScaleTreeGraph.traverse(root, x->x, symbol="Root", filter_fun = MultiScaleTreeGraph.isleaf) +end + +function get_roots_count(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x->x, symbol="Root")) +end + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end + +PlantSimEngine.@process "organ_emergence" verbose = false + +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T + water_leaf_threshold::T +end + +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0,leaves_max_surface_area=100.0, +water_leaf_threshold=30.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area, water_leaf_threshold) + +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0,water_stock=0.0, carbon_stock=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) + +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if water levels are low, prioritise roots + if status.water_stock < m.water_leaf_threshold + return nothing + end + + # if not enough carbon, no organ creation + if status.carbon_stock < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end + +############################ +# Naive water absorption model +# Absorbs precipitation water depending on quantity of roots +############################ +PlantSimEngine.@process "water_absorption" verbose = false + +struct ToyWaterAbsorptionModel <: AbstractWater_AbsorptionModel +end + +PlantSimEngine.inputs_(::ToyWaterAbsorptionModel) = (root_water_assimilation=1.0,) +PlantSimEngine.outputs_(::ToyWaterAbsorptionModel) = (water_absorbed=0.0,) + +function PlantSimEngine.run!(m::ToyWaterAbsorptionModel, models, status, meteo, constants=nothing, extra=nothing) + #root_end = get_root_end_node(status.node) + #root_len = root_end[:Root_len] + status.water_absorbed = meteo.Precipitations * status.root_water_assimilation #* root_len +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsObjectIndependent() + + +########################## +### Root growth : when water stocks are low, expand root +########################## + +PlantSimEngine.@process "root_growth" verbose = false + +struct ToyRootGrowthModel{T} <: AbstractRoot_GrowthModel + water_threshold::T + carbon_root_creation_cost::T + root_max_len::Int +end + +PlantSimEngine.inputs_(::ToyRootGrowthModel) = (water_stock=0.0,carbon_stock=0.0,) +PlantSimEngine.outputs_(::ToyRootGrowthModel) = (carbon_root_creation_consumed=0.0,) + +function PlantSimEngine.run!(m::ToyRootGrowthModel, models, status, meteo, constants=nothing, extra=nothing) + if status.water_stock < m.water_threshold && status.carbon_stock > m.carbon_root_creation_cost + + root_end = get_root_end_node(status.node) + + if length(root_end) != 1 + throw(AssertionError("Couldn't find MTG leaf node with symbol \"Root\"")) + end + root_len = get_roots_count(root_end[1]) + if root_len < m.root_max_len + st = add_organ!(root_end[1], extra, "<", "Root", 2, index=1) + status.carbon_root_creation_consumed = m.carbon_root_creation_cost + end + else + status.carbon_root_creation_consumed = 0.0 + end +end + +########################## +### Model accumulating carbon and water resources +########################## + +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end +#status.water_stock += meteo.precipitations * root_water_assimilation_ratio + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(water_absorbed=0.0,carbon_captured=0.0,carbon_organ_creation_consumed=0.0,carbon_root_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (water_stock=-Inf,carbon_stock=-Inf) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.water_stock += sum(status.water_absorbed) #- status.water_transpiration + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) - sum(status.carbon_root_creation_consumed) + + if status.water_stock < 0.0 + status.water_stock = 0.0 + end +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyStockComputationModel}) = PlantSimEngine.IsObjectIndependent() + +######################## +## Leaf model capturing some arbitrary carbon quantity +######################## + +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple()#(TT_cu=-Inf) +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + # very crude approximation with LAI of 1 and constant PPFD + status.carbon_captured = 200.0 *(1.0 - exp(-0.2)) +end + +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsObjectIndependent() +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyLeafCarbonCaptureModel}) = PlantSimEngine.IsTimeStepIndependent() + +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + :carbon_root_creation_consumed=>["Root"], + :carbon_organ_creation_consumed=>["Internode"] + + ], + ), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + PreviousTimeStep(:water_stock)=>"Plant", + PreviousTimeStep(:carbon_stock)=>"Plant"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Root" => ( MultiScaleModel( + model=ToyRootGrowthModel(10.0, 50.0, 10), + mapped_variables=[PreviousTimeStep(:carbon_stock)=>"Plant", + PreviousTimeStep(:water_stock)=>"Plant"], + ), + ToyWaterAbsorptionModel(), + Status(carbon_root_creation_consumed=0.0, root_water_assimilation=1.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) + + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + plant_root_start = MultiScaleTreeGraph.Node( + plant, + MultiScaleTreeGraph.NodeMTG("+", "Root", 1, 3), + ) + + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + + outs = run!(mtg, mapping, meteo_day) + mtg + + + length(MultiScaleTreeGraph.traverse(mtg,x->x, symbol="Leaf")) \ No newline at end of file diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl new file mode 100644 index 000000000..f28b3cdb6 --- /dev/null +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation3.jl @@ -0,0 +1,251 @@ +########################################### +# Toy plant model with an updated decision model for organ growth +# Physiologically and physically completely meaningless +# (no dimension for units, arbitrary values, stores water and carbon in abstract stocks, +# arbitrary max leaf count and root length, constant and non-coupled photosynthesis and water absorption, ...) +# But it should illustrate the basics of simulating a growing multiscale plant with PlantSimEngine's model approach +########################################### + +function get_root_end_node(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return MultiScaleTreeGraph.traverse(root, x->x, symbol="Root", filter_fun = MultiScaleTreeGraph.isleaf) +end + +function get_roots_count(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + return length(MultiScaleTreeGraph.traverse(root, x->x, symbol="Root")) +end + +function get_n_leaves(node::MultiScaleTreeGraph.Node) + root = MultiScaleTreeGraph.get_root(node) + nleaves = length(MultiScaleTreeGraph.traverse(root, x->1, symbol="Leaf")) + return nleaves +end + +PlantSimEngine.@process "organ_emergence" verbose = false + +struct ToyCustomInternodeEmergence{T} <: AbstractOrgan_EmergenceModel + TT_emergence::T + carbon_internode_creation_cost::T + leaf_surface_area::T + leaves_max_surface_area::T + water_leaf_threshold::T +end + +ToyCustomInternodeEmergence(;TT_emergence=300.0, carbon_internode_creation_cost=200.0, leaf_surface_area=3.0,leaves_max_surface_area=100.0, +water_leaf_threshold=30.0) = ToyCustomInternodeEmergence(TT_emergence, carbon_internode_creation_cost, leaf_surface_area, leaves_max_surface_area, water_leaf_threshold) + +PlantSimEngine.inputs_(m::ToyCustomInternodeEmergence) = (TT_cu=0.0,water_stock=0.0, carbon_stock=0.0, carbon_root_creation_consumed=0.0) +PlantSimEngine.outputs_(m::ToyCustomInternodeEmergence) = (TT_cu_emergence=0.0, carbon_organ_creation_consumed=0.0) + +function PlantSimEngine.run!(m::ToyCustomInternodeEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + leaves_surface_area = m.leaf_surface_area * get_n_leaves(status.node) + status.carbon_organ_creation_consumed = 0.0 + + if leaves_surface_area > m.leaves_max_surface_area + return nothing + end + + # if water levels are low, prioritise roots + if status.water_stock < m.water_leaf_threshold + return nothing + end + + # take into account that the stock may already be depleted + carbon_stock_updated_after_roots = status.carbon_stock - status.carbon_root_creation_consumed + + # if not enough carbon, no organ creation + if carbon_stock_updated_after_roots < m.carbon_internode_creation_cost + return nothing + end + + if length(MultiScaleTreeGraph.children(status.node)) == 2 && + status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + + status_new_internode.TT_cu_emergence = m.TT_emergence - status.TT_cu + status.carbon_organ_creation_consumed = m.carbon_internode_creation_cost + end + + return nothing +end + +############################ +# Naive water absorption model +# Absorbs precipitation water depending on quantity of roots +############################ +PlantSimEngine.@process "water_absorption" verbose = false + +struct ToyWaterAbsorptionModel <: AbstractWater_AbsorptionModel +end + +PlantSimEngine.inputs_(::ToyWaterAbsorptionModel) = (root_water_assimilation=1.0,) +PlantSimEngine.outputs_(::ToyWaterAbsorptionModel) = (water_absorbed=0.0,) + +function PlantSimEngine.run!(m::ToyWaterAbsorptionModel, models, status, meteo, constants=nothing, extra=nothing) + #root_end = get_root_end_node(status.node) + #root_len = root_end[:Root_len] + status.water_absorbed = meteo.Precipitations * status.root_water_assimilation #* root_len +end + +PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsTimeStepIndependent() +PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyWaterAbsorptionModel}) = PlantSimEngine.IsObjectIndependent() + + +########################## +### Root growth : when water stocks are low, expand root +########################## + +PlantSimEngine.@process "root_growth" verbose = false + +struct ToyRootGrowthModel{T} <: AbstractRoot_GrowthModel + carbon_root_creation_cost::T + root_max_len::Int +end + +PlantSimEngine.inputs_(::ToyRootGrowthModel) = NamedTuple() +PlantSimEngine.outputs_(::ToyRootGrowthModel) = (carbon_root_creation_consumed=0.0,) + +function PlantSimEngine.run!(m::ToyRootGrowthModel, models, status, meteo, constants=nothing, extra=nothing) + status.carbon_root_creation_consumed = 0.0 + + root_end = get_root_end_node(status.node) + + if length(root_end) != 1 + throw(AssertionError("Couldn't find MTG leaf node with symbol \"Root\"")) + end + + root_len = get_roots_count(root_end[1]) + if root_len < m.root_max_len + st = add_organ!(root_end[1], extra, "<", "Root", 2, index=1) + status.carbon_root_creation_consumed = m.carbon_root_creation_cost + end +end + +########################## +### Decision model controlling the root growth model +########################## +PlantSimEngine.@process "root_growth_decision" verbose = false + +struct ToyRootGrowthDecisionModel{T} <: AbstractRoot_Growth_DecisionModel + water_threshold::T + carbon_root_creation_cost::T +end + +PlantSimEngine.inputs_(::ToyRootGrowthDecisionModel) = +(water_stock=0.0,carbon_stock=0.0) + +PlantSimEngine.outputs_(::ToyRootGrowthDecisionModel) = NamedTuple() + +PlantSimEngine.dep(::ToyRootGrowthDecisionModel) = (root_growth=AbstractRoot_GrowthModel=>["Root"],) + +function PlantSimEngine.run!(m::ToyRootGrowthDecisionModel, models, status, meteo, constants=nothing, extra=nothing) + + if status.water_stock < m.water_threshold && status.carbon_stock > m.carbon_root_creation_cost + status_Root= extra.statuses["Root"][1] + PlantSimEngine.run!(extra.models["Root"].root_growth, models, status_Root, meteo, constants, extra) + end +end + + +########################## +### Model accumulating carbon and water resources +########################## + +PlantSimEngine.@process "resource_stock_computation" verbose = false + +struct ToyStockComputationModel <: AbstractResource_Stock_ComputationModel +end + +PlantSimEngine.inputs_(::ToyStockComputationModel) = +(water_absorbed=0.0,carbon_captured=0.0,carbon_organ_creation_consumed=0.0,carbon_root_creation_consumed=0.0) + +PlantSimEngine.outputs_(::ToyStockComputationModel) = (water_stock=-Inf,carbon_stock=-Inf) + +function PlantSimEngine.run!(m::ToyStockComputationModel, models, status, meteo, constants=nothing, extra=nothing) + status.water_stock += sum(status.water_absorbed) + status.carbon_stock += sum(status.carbon_captured) - sum(status.carbon_organ_creation_consumed) - sum(status.carbon_root_creation_consumed) +end + + +######################## +## Leaf model capturing some arbitrary carbon quantity +######################## + +PlantSimEngine.@process "leaf_carbon_capture" verbose = false + +struct ToyLeafCarbonCaptureModel<: AbstractLeaf_Carbon_CaptureModel end + +function PlantSimEngine.inputs_(::ToyLeafCarbonCaptureModel) + NamedTuple() +end + +function PlantSimEngine.outputs_(::ToyLeafCarbonCaptureModel) + (carbon_captured=0.0,) +end + +function PlantSimEngine.run!(::ToyLeafCarbonCaptureModel, models, status, meteo, constants, extra) + # very crude approximation with LAI of 1 and constant PPFD + status.carbon_captured = 200.0 *(1.0 - exp(-0.2)) +end + + +mapping = Dict( +"Scene" => ToyDegreeDaysCumulModel(), +"Plant" => ( + MultiScaleModel( + model=ToyStockComputationModel(), + mapped_variables=[ + :carbon_captured=>["Leaf"], + :water_absorbed=>["Root"], + PreviousTimeStep(:carbon_root_creation_consumed)=>"Root", + PreviousTimeStep(:carbon_organ_creation_consumed)=>["Internode"], + ], + ), + ToyRootGrowthDecisionModel(10.0, 50.0), + Status(water_stock = 0.0, carbon_stock = 0.0) + ), +"Internode" => ( + MultiScaleModel( + model=ToyCustomInternodeEmergence(),#TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene", + :water_stock=>"Plant", + :carbon_stock=>"Plant", + :carbon_root_creation_consumed=>"Root"], + ), + Status(carbon_organ_creation_consumed=0.0), + ), +"Root" => (ToyRootGrowthModel(50.0,10), + ToyWaterAbsorptionModel(), + Status(carbon_root_creation_consumed=0.0, root_water_assimilation=1.0), + ), +"Leaf" => ( ToyLeafCarbonCaptureModel(),), +) + + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + + plant = MultiScaleTreeGraph.Node(mtg, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + + internode1 = MultiScaleTreeGraph.Node(plant, MultiScaleTreeGraph.NodeMTG("/", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + internode2 = MultiScaleTreeGraph.Node(internode1, MultiScaleTreeGraph.NodeMTG("<", "Internode", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + MultiScaleTreeGraph.Node(internode2, MultiScaleTreeGraph.NodeMTG("+", "Leaf", 1, 2)) + + plant_root_start = MultiScaleTreeGraph.Node( + plant, + MultiScaleTreeGraph.NodeMTG("+", "Root", 1, 3), + ) + + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + + outs = run!(mtg, mapping, meteo_day) + mtg + + + length(MultiScaleTreeGraph.traverse(mtg,x->x, symbol="Leaf")) \ No newline at end of file diff --git a/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation4.jl b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation4.jl new file mode 100644 index 000000000..7cdcb1b50 --- /dev/null +++ b/examples/ToyMultiScalePlantTutorial/ToyPlantSimulation4.jl @@ -0,0 +1,148 @@ +########################################### +# Toy plant model MTG visualisation using PlantGeom +########################################### +using PlantSimEngine + +using MultiScaleTreeGraph +using PlantSimEngine.Examples +using Pkg +Pkg.add("CSV") +using CSV +include("ToyPlantSimulation3.jl") + +using Plots +using PlantGeom +# reusing the mtg from part 3: +RecipesBase.plot(mtg) + +#= +using GLMakie +#using CairoMakie +using PlantGeom + +PlantGeom.diagram(mtg)=# + + +using PlantGeom.Meshes + +# Internodes and roots will use a cylinder as a mesh + +cylinder() = Meshes.CylinderSurface(1.0) |> Meshes.discretize |> Meshes.simplexify + +refmesh_internode = PlantGeom.RefMesh("Internode", cylinder()) +refmesh_root = PlantGeom.RefMesh("Root", cylinder()) + +# Leaves and petioles are a single mesh, read from a .ply file + +Pkg.add("PlyIO") +using PlyIO +function read_ply(fname) + ply = PlyIO.load_ply(fname) + x = ply["vertex"]["x"] + y = ply["vertex"]["y"] + z = ply["vertex"]["z"] + points = Meshes.Point.(x, y, z) + connec = [Meshes.connect(Tuple(c .+ 1)) for c in ply["face"]["vertex_indices"]] + Meshes.SimpleMesh(points, connec) +end + +leaf_ply = read_ply("examples/leaf_with_petiole.ply") +refmesh_leaf = PlantGeom.RefMesh("Leaf", leaf_ply) + +Pkg.add("TransformsBase") +Pkg.add("Rotations") +#using PlantGeom.TranformsBase +import TransformsBase: → +import Rotations: RotY, RotZ, RotX +# Add the geometry to the MTG, with transformations +function add_geometry!(mtg, refmesh_internode) + + # incremental offset + internode_height = 0.0 + + # relative scale of the base mesh + internode_width = 0.5 + + # length of the base mesh + internode_length = 1.0 + + traverse!(mtg) do node + if symbol(node) == "Internode" + # Set to scale, then translate by the total height + mesh_transformation = Meshes.Scale(internode_width, internode_width, internode_length) → Meshes.Translate(0.0, 0.0, internode_height) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_internode, transformation=mesh_transformation) + + internode_height += internode_length + end + end +end + +add_geometry!(mtg, refmesh_internode) + +# Visualize the mesh +using GLMakie +viz(mtg) + +function add_geometry!(mtg, refmesh_internode, refmesh_root, refmesh_leaf) + + # incremental offset + internode_height = 0.0 + root_depth = 0.0 + + # relative scale of the base mesh + internode_width = 0.5 + root_width = 0.2 + + # length of the base mesh + internode_length = 1.0 + root_length = 1.0 + + # ad hoc value to adjust the base mesh to the scene scale + leaf_mesh_scale = 25 + leaf_scale_width = 0.4*leaf_mesh_scale + leaf_scale_height = 0.4*leaf_mesh_scale + + # Helpers to make the leaves opposite decussate + leaf_rotation = MathConstants.pi / 2.0 + i = 0 + + traverse!(mtg) do node + if symbol(node) == "Internode" + # Set to scale, then translate by the total height + mesh_transformation = Meshes.Scale(internode_width, internode_width, internode_length) → Meshes.Translate(0.0, 0.0, internode_height) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_internode, transformation=mesh_transformation) + + internode_height += node_length + + # Leaves are placed relatively to the parent internode + for chnode in children(node) + if symbol(chnode) == "Leaf" + # Leaves are placed halfway along the the parent internode + mesh_transformation = Meshes.Scale(leaf_scale_width, leaf_scale_width, leaf_scale_height) → Meshes.Rotate(RotX(-MathConstants.pi / 6.0)) → Meshes.Translate(0.0, -internode_width, internode_height - internode_length / 2.0) → Meshes.Rotate(RotZ(leaf_rotation)) + chnode.geometry = PlantGeom.Geometry(ref_mesh=refmesh_leaf, transformation=mesh_transformation) + # Set the second leaf in a pair opposite to the first one => add a 180° rotation + leaf_rotation += MathConstants.pi + end + end + + # Opposite decussate => 90° rotation between pairs + i += 1 + if i % 2 == 0 + leaf_rotation = MathConstants.pi / 2.0 + else + leaf_rotation = MathConstants.pi + end + + elseif symbol(node) == "Root" + mesh_transformation = Meshes.Scale(root_width, root_width, root_length) → Meshes.Translate(0.0, 0.0, root_depth) → Meshes.Rotate(RotZ(MathConstants.pi)) + node.geometry = PlantGeom.Geometry(ref_mesh=refmesh_root, transformation=mesh_transformation) + root_depth -= root_length + end + end +end + +add_geometry!(mtg, refmesh_internode, refmesh_root, refmesh_leaf) + +# Visualize the mesh +using GLMakie +viz(mtg) \ No newline at end of file diff --git a/examples/ToySingleToMultiScale.jl b/examples/ToySingleToMultiScale.jl new file mode 100644 index 000000000..a65a90573 --- /dev/null +++ b/examples/ToySingleToMultiScale.jl @@ -0,0 +1,120 @@ +############################## +### Example single- to multi-scale conversion +############################## + +# Environment setup +using CSV +using DataFrames +using PlantSimEngine +using PlantMeteo +using PlantSimEngine.Examples +using MultiScaleTreeGraph + +# Weather data for all simulations +meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + +############################## +### Single-scale simulation +############################## + +models_singlescale = ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + status=(TT_cu=cumsum(meteo_day.TT),), +) + +outputs_singlescale = run!(models_singlescale, meteo_day) + +############################## +#### Direct translation of the single-scale simulation +############################## +mapping_pseudo_multiscale = Dict( +"Plant" => ( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(0.2), + Status(TT_cu=cumsum(meteo_day.TT),) + ), +) + +mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Plant", 1, 0),) + +# will generate an error as vectors can't be directly passed into a Status in multi-scale simulations +out_pseudo_multiscale = run!(mtg, mapping_pseudo_multiscale, meteo_day) + +############################## +#### Ad Hoc Cumulated Thermal Time Model +############################## + +PlantSimEngine.@process "tt_cu" verbose = false + +struct ToyTt_CuModel <: AbstractTt_CuModel +end + +function PlantSimEngine.run!(::ToyTt_CuModel, models, status, meteo, constants, extra=nothing) + status.TT_cu += + meteo.TT +end + +function PlantSimEngine.inputs_(::ToyTt_CuModel) + NamedTuple() +end + +function PlantSimEngine.outputs_(::ToyTt_CuModel) + (TT_cu=-Inf,) +end + +############################## +#### Actual multiscale version of the single-scale simulation +############################## + +mapping_multiscale = Dict( + "Scene" => ( + ToyTt_CuModel(), + Status(TT_cu=0.0), + ), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + Beer(0.5), + ToyRUEGrowthModel(0.2), + ), +) + +# We now need two nodes for our MTG +mtg_multiscale = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Scene", 1, 0)) + plant = MultiScaleTreeGraph.Node(mtg_multiscale, MultiScaleTreeGraph.NodeMTG("+", "Plant", 1, 1)) + outputs_multiscale = run!(mtg_multiscale, mapping_multiscale, meteo_day) + +############################## +#### Output comparison +############################## + +computed_TT_cu_multiscale = collect(Base.Iterators.flatten(outputs_multiscale["Scene"][:TT_cu])) + +is_approx_equal_1 = true + +for i in 1:length(computed_TT_cu_multiscale) + if !(computed_TT_cu_multiscale[i] ≈ outputs_singlescale.TT_cu[i]) + is_approx_equal_1 = false + break + end +end + +is_approx_equal_1 + +is_approx_equal_2 = length(unique(computed_TT_cu_multiscale .≈ outputs_singlescale.TT_cu)) == 1 + + +# Note : it is also possible to get the weather data length via PlantSimEngine.get_nsteps(meteo_day) +# instead of checking for array length + +is_perfectly_equal = length(unique(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)) == 1 + +(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)[104] +(computed_TT_cu_multiscale .== outputs_singlescale.TT_cu)[105] diff --git a/examples/leaf_with_petiole.ply b/examples/leaf_with_petiole.ply new file mode 100644 index 000000000..7acc9244e Binary files /dev/null and b/examples/leaf_with_petiole.ply differ diff --git a/src/Abstract_model_structs.jl b/src/Abstract_model_structs.jl index 4547b8512..9cef7b087 100644 --- a/src/Abstract_model_structs.jl +++ b/src/Abstract_model_structs.jl @@ -25,4 +25,4 @@ model_(m::AbstractModel) = m get_models(m::AbstractModel) = [model_(m)] # Get the models of an AbstractModel # Note: it is returning a vector of models, because in this case the user provided a single model instead of a vector of. get_status(m::AbstractModel) = nothing -get_mapping(m::AbstractModel) = Pair{Symbol,String}[] \ No newline at end of file +get_mapped_variables(m::AbstractModel) = Pair{Symbol,String}[] \ No newline at end of file diff --git a/src/PlantSimEngine.jl b/src/PlantSimEngine.jl index 1c195d7cd..a6ec6233f 100644 --- a/src/PlantSimEngine.jl +++ b/src/PlantSimEngine.jl @@ -13,7 +13,7 @@ import Term import Markdown # For multi-threading: -import FLoops: @floop, ThreadedEx, SequentialEx, DistributedEx +import FLoops: @floop, @init, ThreadedEx, SequentialEx, DistributedEx # For MTG compatibility: import MultiScaleTreeGraph @@ -107,7 +107,7 @@ export init_status! export add_organ! export @process, process export to_initialize, is_initialized, init_variables, dep -export inputs, outputs, variables +export inputs, outputs, variables, convert_outputs export run! export fit diff --git a/src/checks/dimensions.jl b/src/checks/dimensions.jl index ee6f6373f..b8bd22e62 100644 --- a/src/checks/dimensions.jl +++ b/src/checks/dimensions.jl @@ -37,10 +37,10 @@ w = Weather([ PlantSimEngine.check_dimensions(models, w) # output -ERROR: DimensionMismatch: Component status should have the same number of time-steps (2) than weather data (3). +ERROR: DimensionMismatch: Component status has a vector variable : var1 implying multiple timesteps but weather data only provides a single timestep. ``` """ -check_dimensions(component, weather) = check_dimensions(DataFormat(component), DataFormat(weather), component, weather) +check_dimensions(component, weather) = check_dimensions(DataFormat(weather), component, weather) # Here we add methods for applying to a component, an array or a dict of: function check_dimensions(component::T, w) where {T<:ModelList} @@ -62,19 +62,31 @@ function check_dimensions(component::T, weather) where {T<:AbstractDict{N,<:Mode end -function check_dimensions(::TableAlike, ::TableAlike, st, weather) - length(st) > 1 && length(st) != length(Tables.rows(weather)) && - throw(DimensionMismatch("Component status should have the same number of time-steps ($(length(st))) than weather data ($(length(weather))).")) - return nothing -end +# TODO multi timestep handling # A Status (one time-step) is always authorized with a Weather (it is recycled). # The status is updated at each time-step, but no intermediate saving though! -function check_dimensions(::SingletonAlike, ::TableAlike, st, weather) +function check_dimensions(::TableAlike, st::Status, weather) + weather_len = get_nsteps(weather) + + for (var, value) in zip(keys(st), st) + if length(value) > 1 + if length(value) != weather_len + throw(DimensionMismatch("Component status has a vector variable : $(var) of length $(length(value)) but the weather data expects $(weather_len) timesteps.")) + end + end + end + return nothing end -function check_dimensions(s, ::SingletonAlike, st, weather) +function check_dimensions(::SingletonAlike, st::Status, weather) + for (var, value) in zip(keys(st), st) + if length(value) > 1 + throw(DimensionMismatch("Component status has a vector variable : $(var) implying multiple timesteps but weather data only provides a single timestep.")) + end + end + return nothing end diff --git a/src/component_models/ModelList.jl b/src/component_models/ModelList.jl index 559148f45..b0e23be6a 100644 --- a/src/component_models/ModelList.jl +++ b/src/component_models/ModelList.jl @@ -3,7 +3,6 @@ ModelList(models::M, status::S) ModelList(; status=nothing, - init_fun::Function=init_fun_default, type_promotion=nothing, variables_check=true, kwargs... @@ -23,8 +22,6 @@ type promotion, time steps handling. implements `getproperty`. - `status`: a structure containing the initializations for the variables of the models. Usually a NamedTuple when given as a kwarg, or any structure that implements the Tables interface from `Tables.jl` (*e.g.* DataFrame, see details). -- `nsteps=nothing`: the number of time steps to pre-allocated. If `nothing`, the number of time steps is deduced from the status (or 1 if no status is given). -- `init_fun`: a function that initializes the status based on a vector of NamedTuples (see details). - `type_promotion`: optional type conversion for the variables with default values. `nothing` by default, *i.e.* no conversion. Note that conversion is not applied to the variables input by the user as `kwargs` (need to do it manually). @@ -34,13 +31,6 @@ Should be provided as a Dict with current type as keys and new type as values. # Details -The argument `init_fun` is set by default to `init_fun_default` which initializes the status with a `TimeStepTable` -of `Status` structures. - -If you change `init_fun` by another function, make sure the type you are using (*i.e.* in place of `TimeStepTable`) -implements the `Tables.jl` interface (*e.g.* DataFrame does). And if you still use `TimeStepTable` but only change -`Status`, make sure the type you give is indexable using the dot synthax (*e.g.* `x.var`). - If you need to input a custom Type for the status and make your users able to only partially initialize the `status` field in the input, you'll have to implement a method for `add_model_vars!`, a function that adds the models variables to the type in case it is not fully initialized. The default method is compatible @@ -71,7 +61,7 @@ julia> models = ModelList(process1=Process1Model(1.0), process2=Process2Model(), ```jldoctest 1 julia> typeof(models) -ModelList{@NamedTuple{process1::Process1Model, process2::Process2Model, process3::Process3Model}, TimeStepTable{Status{(:var5, :var4, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}}, Tuple{}} +ModelList{@NamedTuple{process1::Process1Model, process2::Process2Model, process3::Process3Model}, Status{(:var5, :var4, :var6, :var1, :var3, :var2), NTuple{6, Base.RefValue{Float64}}}} ``` No variables were given as keyword arguments, that means that the status of the ModelList is not @@ -97,11 +87,18 @@ julia> meteo = Atmosphere(T = 22.0, Wind = 0.8333, P = 101.325, Rh = 0.4490995); ``` ```jldoctest 1 -julia> run!(models,meteo) +julia> outputs_sim = run!(models,meteo) +TimeStepTable{Status{(:var5, :var4, :var6, ...}(1 x 6): +╭─────┬─────────┬─────────┬─────────┬─────────┬─────────┬─────────╮ +│ Row │ var5 │ var4 │ var6 │ var1 │ var3 │ var2 │ +│ │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ +├─────┼─────────┼─────────┼─────────┼─────────┼─────────┼─────────┤ +│ 1 │ 36.0139 │ 22.0 │ 58.0139 │ 15.0 │ 5.5 │ 0.3 │ +╰─────┴─────────┴─────────┴─────────┴─────────┴─────────┴─────────╯ ``` ```jldoctest 1 -julia> models[:var6] +julia> outputs_sim[:var6] 1-element Vector{Float64}: 58.0138985 ``` @@ -146,52 +143,21 @@ julia> [typeof(models[i][1]) for i in keys(status(models))] Float32 Float32 ``` - -We can also use DataFrame as the status type: - -```jldoctest 1 -julia> using DataFrames; -``` - -```jldoctest 1 -julia> df = DataFrame(:var1 => [13.747, 13.8], :var2 => [1.0, 1.0]); -``` - -```jldoctest 1 -julia> m = ModelList(process1=Process1Model(1.0), process2=Process2Model(), process3=Process3Model(), status=df, init_fun=x -> DataFrame(x)); -``` - -Note that we use `init_fun` to force the status into a `DataFrame`, otherwise it would -be automatically converted into a `TimeStepTable{Status}`. - -```jldoctest 1 -julia> status(m) -2×6 DataFrame - Row │ var5 var4 var6 var1 var3 var2 - │ Float64 Float64 Float64 Float64 Float64 Float64 -─────┼────────────────────────────────────────────────────── - 1 │ -Inf -Inf -Inf 13.747 -Inf 1.0 - 2 │ -Inf -Inf -Inf 13.8 -Inf 1.0 -``` - -Note that computations will be slower using DataFrame, so if performance is an issue, use -TimeStepTable instead (or a NamedTuple as shown in the example). """ -struct ModelList{M<:NamedTuple,S,V<:Tuple{Vararg{Symbol}}} +struct ModelList{M<:NamedTuple,S} models::M status::S - vars_not_propagated::V + type_promotion::Union{Nothing, Dict} end -function ModelList(models::M, status::S) where {M<:NamedTuple{names,T} where {names,T<:NTuple{N,<:AbstractModel} where {N}},S} - ModelList(models, status, ()) -end +#=function ModelList(models::M, status::Status) where {M<:NamedTuple{names,T} where {names,T<:NTuple{N,<:AbstractModel} where {N}}} + ModelList(models, status) +end=# # General interface: function ModelList( args...; status=nothing, - init_fun::Function=init_fun_default, type_promotion::Union{Nothing,Dict}=nothing, variables_check::Bool=true, nsteps=nothing, @@ -218,35 +184,25 @@ function ModelList( mods = merge(args, kwargs) # Make a vector of NamedTuples from the input (please implement yours if you need it) - ts_kwargs = homogeneous_ts_kwargs(status, nsteps) - - # Variables for which a value was given for each time-step by the user: - vars_not_propagated = get_vars_not_propagated(status) - # Note: that the length was checked in homogeneous_ts_kwargs, so we don't need to check it again here. - # Note 2: we need to know these variables because they will not be propagated between time-steps, but set at - # the given value instead. - - # Add the missing variables required by the models (set to default value): - ts_kwargs = add_model_vars(ts_kwargs, mods, type_promotion; init_fun=init_fun, nsteps=nsteps) + ts_kwargs = homogeneous_ts_kwargs(status) + ts_kwargs = add_model_vars(ts_kwargs, mods, type_promotion) model_list = ModelList( mods, ts_kwargs, - vars_not_propagated + type_promotion ) variables_check && !is_initialized(model_list) return model_list end -parse_models(m) = NamedTuple([process(i) => i for i in m]) +outputs(m::ModelList) = m.outputs -init_fun_default(x::Vector{T}) where {T} = TimeStepTable([Status(i) for i in x]) -init_fun_default(x::N) where {N<:NamedTuple} = TimeStepTable([Status(x)]) -init_fun_default(x) = x +parse_models(m) = NamedTuple([process(i) => i for i in m]) """ - add_model_vars(x, models, type_promotion; init_fun=init_fun_default) + add_model_vars(x, models, type_promotion) Check which variables in `x` are not initialized considering a set of `models` and the variables needed for their simulation. If some variables are uninitialized, initialize them to their default values. @@ -256,14 +212,15 @@ any Tables.jl-compatible `x` and for NamedTuples. Careful, the function makes a copy of the input `x` if it does not list all needed variables. """ -function add_model_vars(x, models, type_promotion; init_fun=init_fun_default, nsteps=nothing) +function add_model_vars(x, models, type_promotion) ref_vars = merge(init_variables(models; verbose=false)...) # If no variable is required, we return the input: - length(ref_vars) == 0 && return x + length(ref_vars) == 0 && return isa(x, Status) ? x : Status(x) # If the user gave a status, we check if all the variables are already initialized: vars_in_x = status_keys(x) - all([k in vars_in_x for k in keys(ref_vars)]) && return x # If so, we return the input + status_x = + all([k in vars_in_x for k in keys(ref_vars)]) && return isa(x, Status) ? x : Status(x) # If so, we return the input # Else, we add the variables by making a new object (carefull, this is a copy so it takes more time): @@ -271,25 +228,24 @@ function add_model_vars(x, models, type_promotion; init_fun=init_fun_default, ns ref_vars = convert_vars(ref_vars, type_promotion) # If the user gave an empty status, we initialize all variables to their default values: - if x === nothing || (!Tables.istable(x) && length(x) == 0) - if nsteps === nothing - return init_fun(ref_vars) - else - return init_fun(fill(ref_vars, nsteps)) - end + if x === nothing + return Status(ref_vars) end - + if Tables.istable(x) - # Making a vars for each ith value in the user vars: - x_full = [merge(ref_vars, NamedTuple(Tables.rows(x)[1]))] - for r in Tables.rows(x)[2:end] - push!(x_full, merge(ref_vars, NamedTuple(r))) - end + # This situation only occurs if the user provided a table instead of a status + # Meaning we have a status of vector values, all initialized up to a certain point + # Unsure this is desirable, as that means run! does nothing or overwrites everything + # Anyway, we wish to create a NamedTuple() of Vectors here + x_full = (;zip(propertynames(x), Tables.columns(x))...) + x_full = merge(ref_vars, x_full) + else x_full = merge(ref_vars, NamedTuple(x)) end + #x_full = merge(ref_vars, NamedTuple(x)) - return init_fun(x_full) + return Status(x_full) end function status_keys(st) @@ -304,7 +260,7 @@ function add_model_vars(x::Nothing, models, type_promotion) ref_vars = merge(init_variables(models; verbose=false)...) length(ref_vars) == 0 && return x # Convert model variables types to the one required by the user: - return convert_vars(ref_vars, type_promotion) + return Status(convert_vars(ref_vars, type_promotion)) end """ @@ -312,7 +268,7 @@ end By default, the function returns its argument. """ -homogeneous_ts_kwargs(kwargs, nsteps) = kwargs +homogeneous_ts_kwargs(kwargs) = kwargs """ kwargs_to_timestep(kwargs::NamedTuple{N,T}) where {N,T} @@ -327,42 +283,15 @@ It is used to be able to *e.g.* give constant values for all time-steps for one PlantSimEngine.homogeneous_ts_kwargs((Tₗ=[25.0, 26.0], aPPFD=1000.0)) ``` """ -function homogeneous_ts_kwargs(kwargs::NamedTuple{N,T}, nsteps) where {N,T} +function homogeneous_ts_kwargs(kwargs::NamedTuple{N,T}) where {N,T} length(kwargs) == 0 && return kwargs vars_vals = collect(Any, values(kwargs)) - length_vars = [isa(i, RefVector) ? 1 : length(i) for i in vars_vals] - #Note: length is 1 for RefVector because it is a vector of references to other scales, - # not a vector of values - - # One of the variable is given as an array, meaning this is actually several - # time-steps. In this case we make an array of vars. - max_length_st = nsteps !== nothing ? nsteps : maximum(length_vars) - - for i in eachindex(vars_vals) - # If the ith vars has length one, repeat its value to match the max time-steps: - if length_vars[i] == 1 - vars_vals[i] = repeat([vars_vals[i]], max_length_st) - else - length_vars[i] != max_length_st && @error "$(keys(kwargs)[i]) should be length $max_length_st or 1" - end - end - - # Making a vars for each ith value in the user vars: - vars_array = NamedTuple[NamedTuple{keys(kwargs)}(j[i] for j in vars_vals) for i in 1:max_length_st] + + vars_array = NamedTuple{keys(kwargs)}(j for j in vars_vals) return vars_array end - -""" - get_vars_not_propagated(status) - -Returns all variables that are given for several time-steps in the status. -""" -get_vars_not_propagated(status) = (findall(x -> length(x) > 1, status)...,) -get_vars_not_propagated(df::DataFrames.DataFrame) = (propertynames(df)...,) -get_vars_not_propagated(::Nothing) = () - """ Base.copy(l::ModelList) Base.copy(l::ModelList, status) @@ -396,7 +325,7 @@ function Base.copy(m::T) where {T<:ModelList} ModelList( m.models, deepcopy(m.status), - m.vars_not_propagated + deepcopy(m.type_promotion) ) end @@ -404,7 +333,7 @@ function Base.copy(m::T, status) where {T<:ModelList} ModelList( m.models, status, - m.vars_not_propagated + deepcopy(m.type_promotion) ) end diff --git a/src/component_models/Status.jl b/src/component_models/Status.jl index eed425a44..c09f36403 100644 --- a/src/component_models/Status.jl +++ b/src/component_models/Status.jl @@ -133,46 +133,38 @@ function Base.:(==)(s1::Status, s2::Status) end -""" - propagate_values!(status1::Dict, status2::Dict, vars_not_propagated::Set) - -Propagates the values of all variables in `status1` to `status2`, except for vars in `vars_not_propagated`. - -# Arguments - -- `status1::Dict`: A dictionary containing the current values of variables. -- `status2::Dict`: A dictionary to which the values of variables will be propagated. -- `vars_not_propagated::Set`: A set of variables whose values should not be propagated. - -# Examples - -```jldoctest st1 -julia> status1 = Status(var1 = 15.0, var2 = 0.3); -``` - -```jldoctest st1 -julia> status2 = Status(var1 = 16.0, var2 = -Inf); -``` - -```jldoctest st1 -julia> vars_not_propagated = (:var1,); - -```jldoctest st1 -julia> PlantSimEngine.propagate_values!(status1, status2, vars_not_propagated); -``` +# Returns a status with all vector variables replaced with their first value (ie a Status ready for simulation) +# also returns a tuple of symbols corresponding to the vector variables +function flatten_status(s::Status) + status_values_flattened = NamedTuple() + vector_variables = NamedTuple() + + for (var, value) in zip(keys(s), s) + if length(value) > 1 + vector_variables = (vector_variables..., var) + status_values_flattened = (status_values_flattened..., value[1]) + else + status_values_flattened = (status_values_flattened..., value) + end + end -```jldoctest st1 -julia> status2.var2 == status1.var2 -true -``` + return Status(;zip(keys(s), status_values_flattened)...), vector_variables +end -```jldoctest st1 -julia> status2.var1 == status1.var1 -false -``` -""" -function propagate_values!(status1, status2, vars_not_propagated) - for var in setdiff(keys(status1), vars_not_propagated) - status2[var] = status1[var] +# Update to the next timestep the variables that were passed in as vectors by the user +function update_vector_variables(s::Status, sf::Status, vector_variables, i) + for vec in vector_variables + sf[vec] = s[vec][i] end end + +# TODO do a bit more and return error if there is a length discrepancy that isn't accounted for by timestep differences +function get_status_vector_max_length(s::Status) + max_len = 1 + for (var, value) in zip(keys(s), s) + if length(value) > 1 + max_len = length(value) + end + end + return max_len +end \ No newline at end of file diff --git a/src/dataframe.jl b/src/dataframe.jl index 0904fda8f..a47e67e77 100644 --- a/src/dataframe.jl +++ b/src/dataframe.jl @@ -60,23 +60,11 @@ function DataFrames.DataFrame(components::T) where {T<:AbstractDict{N,<:ModelLis reduce(vcat, df) end -# NB: could use dispatch on concrete types but would enforce specific implementation for each - - -""" - DataFrame(components::ModelList{T,<:TimeStepTable}) - -Implementation of `DataFrame` for a `ModelList` model with several time steps. -""" -function DataFrames.DataFrame(components::ModelList{T,S,V}) where {T,S<:TimeStepTable,V} - DataFrames.DataFrame([(NamedTuple(j)..., timestep=i) for (i, j) in enumerate(status(components))]) -end - """ - DataFrame(components::ModelList{T,S,V}) where {T,S<:Status,V} + DataFrame(components::ModelList{T,S}) where {T,S<:Status} Implementation of `DataFrame` for a `ModelList` model with one time step. """ -function DataFrames.DataFrame(components::ModelList{T,S,V}) where {T,S<:Status,V} +function DataFrames.DataFrame(components::ModelList{T,S}) where {T,S<:Status} DataFrames.DataFrame([NamedTuple(status(components)[1])]) end diff --git a/src/dependencies/hard_dependencies.jl b/src/dependencies/hard_dependencies.jl index 4728b1028..0d1fa5b3c 100644 --- a/src/dependencies/hard_dependencies.jl +++ b/src/dependencies/hard_dependencies.jl @@ -113,7 +113,7 @@ end # When we use a mapping (multiscale), we return the set of soft-dependencies (we put the hard-dependencies as their children): function hard_dependencies(mapping::Dict{String,T}; verbose::Bool=true) where {T} - full_vars_mapping = Dict(first(mod) => Dict(get_mapping(last(mod))) for mod in mapping) + full_vars_mapping = Dict(first(mod) => Dict(get_mapped_variables(last(mod))) for mod in mapping) soft_dep_graphs = Dict{String,Any}() not_found = Dict{Symbol,DataType}() diff --git a/src/doc_templates/mtg-related.jl b/src/doc_templates/mtg-related.jl index 238abfc36..c5c3666dc 100644 --- a/src/doc_templates/mtg-related.jl +++ b/src/doc_templates/mtg-related.jl @@ -34,7 +34,7 @@ mapping = Dict( \ "Plant" => ( \ MultiScaleModel( \ model=ToyCAllocationModel(), \ - mapping=[ \ + mapped_variables=[ \ :carbon_assimilation => ["Leaf"], \ :carbon_demand => ["Leaf", "Internode"], \ :carbon_allocation => ["Leaf", "Internode"] \ @@ -42,7 +42,7 @@ mapping = Dict( \ ), MultiScaleModel( \ model=ToyPlantRmModel(), \ - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],] \ + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],] \ ), \ ),\ "Internode" => ( \ @@ -53,7 +53,7 @@ mapping = Dict( \ "Leaf" => ( \ MultiScaleModel( \ model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ + mapped_variables=[:soil_water_content => "Soil",], \ ), \ ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), \ diff --git a/src/mtg/GraphSimulation.jl b/src/mtg/GraphSimulation.jl index 4e49bbd92..9f243aeb3 100644 --- a/src/mtg/GraphSimulation.jl +++ b/src/mtg/GraphSimulation.jl @@ -40,18 +40,19 @@ get_models(g::GraphSimulation) = g.models outputs(g::GraphSimulation) = g.outputs """ - outputs(sim::GraphSimulation, sink) + convert_outputs(sim_outputs::Dict{String,O} where O, sink; refvectors=false, no_value=nothing) + convert_outputs(sim_outputs::TimeStepTable{T} where T, sink) -Get the outputs from a simulation made on a plant graph. +Convert the outputs returned by a simulation made on a plant graph into another format. # Details -The first method returns a vector of `NamedTuple`, the second formats it -sing the sink function, for exemple a `DataFrame`. +The first method operates on the outputs of a multiscale simulation, the second one on those of a typical single-scale simulation. +The sink function determines the format used, for exemple a `DataFrame`. # Arguments -- `sim::GraphSimulation`: the simulation object, typically returned by `run!`. +- `sim_outputs : the outputs of a prior simulation, typically returned by `run!`. - `sink`: a sink compatible with the Tables.jl interface (*e.g.* a `DataFrame`) - `refvectors`: if `false` (default), the function will remove the RefVector values, otherwise it will keep them - `no_value`: the value to replace `nothing` values. Default is `nothing`. Usually used to replace `nothing` values @@ -76,7 +77,7 @@ mtg = import_mtg_example(); ``` ```@example -sim = run!(mtg, mapping, meteo, outputs = Dict( +out = run!(mtg, mapping, meteo, tracked_outputs = Dict( "Leaf" => (:carbon_assimilation, :carbon_demand, :soil_water_content, :carbon_allocation), "Internode" => (:carbon_allocation,), "Plant" => (:carbon_allocation,), @@ -85,13 +86,12 @@ sim = run!(mtg, mapping, meteo, outputs = Dict( ``` ```@example -outputs(sim, DataFrames) +convert_outputs(out, DataFrames) ``` """ -function outputs(sim::GraphSimulation, sink; refvectors=false, no_value=nothing) +function convert_outputs(outs::Dict{String,O} where O, sink; refvectors=false, no_value=nothing) @assert Tables.istable(sink) "The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)" - outs = outputs(sim) variables_names_types = Iterators.flatten(collect(i.first => eltype(i.second[1]) for i in filter(x -> x.first != :node, vars)) for (organs, vars) in outs) |> collect variables_names_types_dict = Dict{Symbol,Any}() @@ -113,6 +113,20 @@ function outputs(sim::GraphSimulation, sink; refvectors=false, no_value=nothing) variables_names_types = (timestep=Int, organ=String, node=Int, NamedTuple(variables_names_types_dict)...) var_names_all = keys(variables_names_types) t = NamedTuple{var_names_all,Tuple{values(variables_names_types)...}}[] + #=size_hint = 0 + for (organ, vars) in outs # organ = "Leaf"; vars = outs[organ] + var_names = setdiff(collect(keys(vars)), [:node]) + if length(var_names) == 0 + continue + end + steps_iterable = axes(vars[var_names[1]], 1) + for timestep in steps_iterable # timestep = 1 + node_iterable = axes(vars[var_names[1]][timestep], 1) + size_hint+=length(node_iterable) + end + end + + sizehint!(t, size_hint)=# for (organ, vars) in outs # organ = "Leaf"; vars = outs[organ] var_names = setdiff(collect(keys(vars)), [:node]) @@ -142,10 +156,16 @@ function outputs(sim::GraphSimulation, sink; refvectors=false, no_value=nothing) return sink(t) end -function outputs(sim::GraphSimulation, key::Symbol) - Tables.columns(outputs(sim, Vector{NamedTuple}))[key] +function outputs(outs::Dict{String, O} where O, key::Symbol) + Tables.columns(convert_outputs(outs, Vector{NamedTuple}))[key] +end + +function outputs(outs::Dict{String, O} where O, i::T) where {T<:Integer} + Tables.columns(convert_outputs(outs, Vector{NamedTuple}))[i] end -function outputs(sim::GraphSimulation, i::T) where {T<:Integer} - Tables.columns(outputs(sim, Vector{NamedTuple}))[i] +# ModelLists now return outputs as a TimeStepTable{Status}, conversion is straightforward +function convert_outputs(out::TimeStepTable{T} where T, sink) + @assert Tables.istable(sink) "The sink argument must be compatible with the Tables.jl interface (`Tables.istable(sink)` must return `true`, *e.g.* `DataFrame`)" + return sink(out) end \ No newline at end of file diff --git a/src/mtg/MultiScaleModel.jl b/src/mtg/MultiScaleModel.jl index 0182e32eb..3243afc6a 100644 --- a/src/mtg/MultiScaleModel.jl +++ b/src/mtg/MultiScaleModel.jl @@ -1,5 +1,5 @@ """ - MultiScaleModel(model, mapping) + MultiScaleModel(model, mapped_variables) A structure to make a model multi-scale. It defines a mapping between the variables of a model and the nodes symbols from which the values are taken from. @@ -7,9 +7,9 @@ model and the nodes symbols from which the values are taken from. # Arguments - `model<:AbstractModel`: the model to make multi-scale -- `mapping<:Vector{Pair{Symbol,Union{AbstractString,Vector{AbstractString}}}}`: a vector of pairs of symbols and strings or vectors of strings +- `mapped_variables<:Vector{Pair{Symbol,Union{AbstractString,Vector{AbstractString}}}}`: a vector of pairs of symbols and strings or vectors of strings -The mapping can be of the form: +The mapped_variables argument can be of the form: 1. `[:variable_name => "Plant"]`: We take one value from the Plant node 2. `[:variable_name => ["Leaf"]]`: We take a vector of values from the Leaf nodes @@ -74,25 +74,25 @@ We can make it multi-scale by defining a mapping between the variables of the mo For example, if the `carbon_allocation` comes from the `Leaf` and `Internode` nodes, we can define the mapping as follows: ```jldoctest mylabel -julia> mapping = [:carbon_allocation => ["Leaf", "Internode"]] +julia> mapped_variables=[:carbon_allocation => ["Leaf", "Internode"]] 1-element Vector{Pair{Symbol, Vector{String}}}: :carbon_allocation => ["Leaf", "Internode"] ``` -The mapping is a vector of pairs of symbols and strings or vectors of strings. In this case, we have only one pair to define the mapping +The mapped_variables argument is a vector of pairs of symbols and strings or vectors of strings. In this case, we have only one pair to define the mapping between the `carbon_allocation` variable and the `Leaf` and `Internode` nodes. -We can now make the model multi-scale by passing the model and the mapping to the `MultiScaleModel` constructor : +We can now make the model multi-scale by passing the model and the mapped variables to the `MultiScaleModel` constructor : ```jldoctest mylabel -julia> multiscale_model = PlantSimEngine.MultiScaleModel(model, mapping) +julia> multiscale_model = PlantSimEngine.MultiScaleModel(model, mapped_variables) MultiScaleModel{ToyCAllocationModel, Vector{Pair{Union{Symbol, PreviousTimeStep}, Union{Pair{String, Symbol}, Vector{Pair{String, Symbol}}}}}}(ToyCAllocationModel(), Pair{Union{Symbol, PreviousTimeStep}, Union{Pair{String, Symbol}, Vector{Pair{String, Symbol}}}}[:carbon_allocation => ["Leaf" => :carbon_allocation, "Internode" => :carbon_allocation]]) ``` -We can access the mapping and the model: +We can access the mapped variables and the model: ```jldoctest mylabel -julia> PlantSimEngine.mapping_(multiscale_model) +julia> PlantSimEngine.mapped_variables_(multiscale_model) 1-element Vector{Pair{Union{Symbol, PreviousTimeStep}, Union{Pair{String, Symbol}, Vector{Pair{String, Symbol}}}}}: :carbon_allocation => ["Leaf" => :carbon_allocation, "Internode" => :carbon_allocation] ``` @@ -104,12 +104,12 @@ ToyCAllocationModel() """ struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Symbol},Vector{Pair{S,Symbol}}}}} where {A<:Union{Symbol,PreviousTimeStep},S<:AbstractString}} model::T - mapping::V + mapped_variables::V - function MultiScaleModel{T}(model::T, mapping) where {T<:AbstractModel} + function MultiScaleModel{T}(model::T, mapped_variables) where {T<:AbstractModel} # Check that the variables in the mapping are variables of the model: model_variables = keys(variables(model)) - for i in mapping + for i in mapped_variables # If the var is a PreviousTimeStep, we take the variable name, else take the first element of the pair: var = isa(i, PreviousTimeStep) ? i.variable : first(i) @@ -133,7 +133,7 @@ struct MultiScaleModel{T<:AbstractModel,V<:AbstractVector{Pair{A,Union{Pair{S,Sy process_ = process(model) unfolded_mapping = Pair{Union{Symbol,PreviousTimeStep},Union{Pair{String,Symbol},Vector{Pair{String,Symbol}}}}[] - for i in mapping + for i in mapped_variables push!(unfolded_mapping, _get_var(isa(i, PreviousTimeStep) ? i : Pair(i.first, i.second), process_)) # Note: We are using Pair(i.first, i.second) to make sure the Pair is specialized enough, because sometimes the vector in the mapping made the Pair not specialized enough e.g. [:v1 => "S" => :v2,:v3 => "S"] makes the pairs `Pair{Symbol, Any}`. end @@ -185,16 +185,16 @@ end -function MultiScaleModel(model::T, mapping) where {T<:AbstractModel} - MultiScaleModel{T}(model, mapping) +function MultiScaleModel(model::T, mapped_variables) where {T<:AbstractModel} + MultiScaleModel{T}(model, mapped_variables) end -MultiScaleModel(; model, mapping) = MultiScaleModel(model, mapping) +MultiScaleModel(; model, mapped_variables) = MultiScaleModel(model, mapped_variables) -mapping_(m::MultiScaleModel) = m.mapping +mapped_variables_(m::MultiScaleModel) = m.mapped_variables model_(m::MultiScaleModel) = m.model inputs_(m::MultiScaleModel) = inputs_(m.model) outputs_(m::MultiScaleModel) = outputs_(m.model) get_models(m::MultiScaleModel) = [model_(m)] # Get the models of a MultiScaleModel: # Note: it is returning a vector of models, because in this case the user provided a single MultiScaleModel instead of a vector of. get_status(m::MultiScaleModel) = nothing -get_mapping(m::MultiScaleModel{T,S}) where {T,S} = mapping_(m) \ No newline at end of file +get_mapped_variables(m::MultiScaleModel{T,S}) where {T,S} = mapped_variables_(m) \ No newline at end of file diff --git a/src/mtg/mapping/getters.jl b/src/mtg/mapping/getters.jl index dc4c4fdb9..4c46a7dbd 100644 --- a/src/mtg/mapping/getters.jl +++ b/src/mtg/mapping/getters.jl @@ -26,7 +26,7 @@ If we just give a MultiScaleModel, we get its model as a one-element vector: ```jldoctest mylabel julia> models = MultiScaleModel( \ model=ToyCAllocationModel(), \ - mapping=[ \ + mapped_variables=[ \ :carbon_assimilation => ["Leaf"], \ :carbon_demand => ["Leaf", "Internode"], \ :carbon_allocation => ["Leaf", "Internode"] \ @@ -46,7 +46,7 @@ If we give a tuple of models, we get each model in a vector: julia> models2 = ( \ MultiScaleModel( \ model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ + mapped_variables=[:soil_water_content => "Soil",], \ ), \ ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ Status(aPPFD=1300.0, TT=10.0), \ @@ -90,7 +90,7 @@ function get_status(m) end """ - get_mapping(m) + get_mapped_variables(m) Get the mapping of a dictionary of model mapping. @@ -104,8 +104,8 @@ Returns a vector of pairs of symbols and strings or vectors of strings See [`get_models`](@ref) for examples. """ -function get_mapping(m) - mod_mapping = [mapping_(i) for i in m if isa(i, MultiScaleModel)] +function get_mapped_variables(m) + mod_mapping = [mapped_variables_(i) for i in m if isa(i, MultiScaleModel)] if length(mod_mapping) == 0 return Pair{Symbol,String}[] end diff --git a/src/mtg/mapping/model_generation_from_status_vectors.jl b/src/mtg/mapping/model_generation_from_status_vectors.jl index 18303b131..ccf83f42b 100644 --- a/src/mtg/mapping/model_generation_from_status_vectors.jl +++ b/src/mtg/mapping/model_generation_from_status_vectors.jl @@ -86,7 +86,7 @@ function replace_mapping_status_vectors_with_generated_models(mapping_with_vecto HelperNextTimestepModel(), MultiScaleModel( model=HelperCurrentTimestepModel(), - mapping=[PreviousTimeStep(:next_timestep),], + mapped_variables=[PreviousTimeStep(:next_timestep),], ), mapping[organ], ) else @@ -94,7 +94,7 @@ function replace_mapping_status_vectors_with_generated_models(mapping_with_vecto HelperNextTimestepModel(), MultiScaleModel( model=HelperCurrentTimestepModel(), - mapping=[PreviousTimeStep(:next_timestep),], + mapped_variables=[PreviousTimeStep(:next_timestep),], ), mapping[organ]..., ) end @@ -165,7 +165,7 @@ function generate_model_from_status_vector_variable(mapping, timestep_scale, sta # if :current_timestep is not in the same scale if timestep_scale != organ - model_add_decl = "generated_models_534f1c161f91bb346feba1a84a55e8251f5ad446 = (generated_models_534f1c161f91bb346feba1a84a55e8251f5ad446..., MultiScaleModel(model=$model_name($value_534f1c161f91bb346feba1a84a55e8251f5ad446), mapping=[:current_timestep=>\"$timestep_scale\"],),)" + model_add_decl = "generated_models_534f1c161f91bb346feba1a84a55e8251f5ad446 = (generated_models_534f1c161f91bb346feba1a84a55e8251f5ad446..., MultiScaleModel(model=$model_name($value_534f1c161f91bb346feba1a84a55e8251f5ad446), mapped_variables=[:current_timestep=>\"$timestep_scale\"],),)" end eval(Meta.parse(model_add_decl)) @@ -191,17 +191,29 @@ function modellist_to_mapping(modellist_original::ModelList, modellist_status; n models = modellist.models - mapping_incomplete = Dict( + mapping_incomplete = isnothing(modellist_status) ? + ( + Dict( default_scale => ( models..., MultiScaleModel( model=HelperCurrentTimestepModel(), - mapping=[PreviousTimeStep(:next_timestep),], + mapped_variables=[PreviousTimeStep(:next_timestep),], + ), + Status((current_timestep=1,next_timestep=1,)) + ), + )) : ( + Dict( + default_scale => ( + models..., + MultiScaleModel( + model=HelperCurrentTimestepModel(), + mapped_variables=[PreviousTimeStep(:next_timestep),], ), Status((modellist_status..., current_timestep=1,next_timestep=1,)) ), ) - + ) timestep_scale = "Default" organ = "Default" @@ -214,7 +226,7 @@ function modellist_to_mapping(modellist_original::ModelList, modellist_status; n HelperNextTimestepModel(), MultiScaleModel( model=HelperCurrentTimestepModel(), - mapping=[PreviousTimeStep(:next_timestep),], + mapped_variables=[PreviousTimeStep(:next_timestep),], ), new_status, ), diff --git a/src/mtg/mapping/reverse_mapping.jl b/src/mtg/mapping/reverse_mapping.jl index 3c36804d7..059b42947 100644 --- a/src/mtg/mapping/reverse_mapping.jl +++ b/src/mtg/mapping/reverse_mapping.jl @@ -34,7 +34,7 @@ julia> mapping = Dict( \ "Plant" => \ MultiScaleModel( \ model=ToyCAllocationModel(), \ - mapping=[ \ + mapped_variables=[ \ :carbon_assimilation => ["Leaf"], \ :carbon_demand => ["Leaf", "Internode"], \ :carbon_allocation => ["Leaf", "Internode"] \ @@ -44,7 +44,7 @@ julia> mapping = Dict( \ "Leaf" => ( \ MultiScaleModel( \ model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ + mapped_variables=[:soil_water_content => "Soil",], \ ), \ ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ Status(aPPFD=1300.0, TT=10.0), \ diff --git a/src/mtg/save_results.jl b/src/mtg/save_results.jl index 2c1afe8e9..a024e1468 100644 --- a/src/mtg/save_results.jl +++ b/src/mtg/save_results.jl @@ -38,7 +38,7 @@ julia> mapping = Dict( \ "Plant" => ( \ MultiScaleModel( \ model=ToyCAllocationModel(), \ - mapping=[ \ + mapped_variables=[ \ :carbon_assimilation => ["Leaf"], \ :carbon_demand => ["Leaf", "Internode"], \ :carbon_allocation => ["Leaf", "Internode"] \ @@ -46,7 +46,7 @@ julia> mapping = Dict( \ ), MultiScaleModel( \ model=ToyPlantRmModel(), \ - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],] \ + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],] \ ), \ ),\ "Internode" => ( \ @@ -57,7 +57,7 @@ julia> mapping = Dict( \ "Leaf" => ( \ MultiScaleModel( \ model=ToyAssimModel(), \ - mapping=[:soil_water_content => "Soil",], \ + mapped_variables=[:soil_water_content => "Soil",], \ ), \ ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), \ ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), \ @@ -111,11 +111,23 @@ julia> collect(keys(preallocated_vars["Leaf"])) """ function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_mapping, vars_need_init, outs, nsteps; type_promotion=nothing, check=true) outs_ = Dict{String,Vector{Symbol}}() - for i in keys(outs) # i = "Plant" - @assert isa(outs[i], Tuple{Vararg{Symbol}}) """Outputs for scale $i should be a tuple of symbols, *e.g.* `"$i" => (:a, :b)`, found `"$i" => $(outs[i])` instead.""" - outs_[i] = [outs[i]...] + + # default behaviour : track everything + if isnothing(outs) + for organ in keys(statuses) + outs_[organ] = [keys(statuses_template[organ])...] + end + # No outputs requested by user : just return the timestep and node + elseif length(outs) == 0 + for i in keys(statuses) + outs_[i] = [] + end + else + for i in keys(outs) # i = "Plant" + @assert isa(outs[i], Tuple{Vararg{Symbol}}) """Outputs for scale $i should be a tuple of symbols, *e.g.* `"$i" => (:a, :b)`, found `"$i" => $(outs[i])` instead.""" + outs_[i] = [outs[i]...] + end end - statuses_ = copy(statuses_template) # Checking that organs in outputs exist in the mtg (in the statuses): if !all(i in keys(statuses) for i in keys(outs_)) @@ -190,7 +202,6 @@ function pre_allocate_outputs(statuses, statuses_template, reverse_multiscale_ma # without the reference types, e.g. RefVector{Float64} becomes Vector{Float64}. end -pre_allocate_outputs(statuses, status_templates, reverse_multiscale_mapping, vars_need_init, ::Nothing, nsteps; type_promotion=nothing, check=true) = Dict{String,Tuple{Symbol,Vararg{Symbol}}}() """ save_results!(object::GraphSimulation, i) @@ -201,6 +212,11 @@ from the `status(object)` in the `outputs(object)`. """ function save_results!(object::GraphSimulation, i) outs = outputs(object) + + if length(outs) == 0 + return + end + statuses = status(object) for (organ, vars) in outs @@ -208,4 +224,62 @@ function save_results!(object::GraphSimulation, i) values[i] = [status[var] for status in statuses[organ]] end end +end + +function pre_allocate_outputs(m::ModelList, outs, nsteps; type_promotion=nothing, check=true) + + # NOTE : init_variables recreates a DependencyGraph, it's not great + # TODO : copy ? + out_vars_pre_type_promotion = merge(init_variables(m; verbose=false)...) + + # bit hacky, could be cleaned up + out_vars_all = convert_vars(out_vars_pre_type_promotion, m.type_promotion) + + out_keys_requested = Symbol[] + if !isnothing(outs) + if length(outs) == 0 # no outputs desired, for some reason + return NamedTuple() + end + out_keys_requested = Symbol[outs...] + end + out_vars_requested = NamedTuple() + + # default implicit behaviour, track everything + if isempty(out_keys_requested) + out_vars_requested = out_vars_all + else + unexpected_outputs = setdiff(out_keys_requested, status_keys(status(m))) + + if !isempty(unexpected_outputs) + e = string( + "You requested as output ", + join(unexpected_outputs, " ,"), + " not found in any model." + ) + + if check + error(e) + else + @info e + [delete!(unexpected_outputs, i) for i in unexpected_outputs] + end + end + + out_defaults_requested = (out_vars_all[i] for i in out_keys_requested) + out_vars_requested = (;zip(out_keys_requested, out_defaults_requested)...) + end + + outputs_timestep = fill(out_vars_requested, nsteps) + return TimeStepTable([Status(i) for i in outputs_timestep]) +end + +function save_results!(status_flattened::Status, outputs, i) + if length(outputs) == 0 + return + end + outs = outputs[i] + + for var in keys(outs) + outs[var] = status_flattened[var] + end end \ No newline at end of file diff --git a/src/processes/model_initialisation.jl b/src/processes/model_initialisation.jl index 971c3a43a..5da66bdc5 100755 --- a/src/processes/model_initialisation.jl +++ b/src/processes/model_initialisation.jl @@ -104,14 +104,11 @@ function to_initialize(m::DependencyGraph) return needed_variables_process end -""" - to_initialize(m::AbstractDependencyNode) -Return the variables that must be initialized providing a set of models and processes. The -function just returns the inputs and outputs of each model, with their default values. -To take into account model coupling, use the function at an upper-level instead, *i.e.* -`to_initialize(m::ModelList)` or `to_initialize(m::DependencyGraph)`. -""" +#Return the variables that must be initialized providing a set of models and processes. The +#function just returns the inputs and outputs of each model, with their default values. +#To take into account model coupling, use the function at an upper-level instead, *i.e.* +# `to_initialize(m::ModelList)` or `to_initialize(m::DependencyGraph)`. function to_initialize(m::AbstractDependencyNode) return (inputs=inputs_(m.value), outputs=outputs_(m.value)) end diff --git a/src/processes/models_inputs_outputs.jl b/src/processes/models_inputs_outputs.jl index 47ffff03a..d003ed15f 100644 --- a/src/processes/models_inputs_outputs.jl +++ b/src/processes/models_inputs_outputs.jl @@ -140,12 +140,6 @@ function variables(m::T, ms...) where {T<:Union{Missing,AbstractModel}} length((ms...,)) > 0 ? merge(variables(m), variables(ms...)) : merge(inputs_(m), outputs_(m)) end -""" - variables(m::AbstractDependencyNode) - -Returns a tuple with the name of the inputs and outputs variables needed by a model in -a dependency graph. -""" function variables(m::SoftDependencyNode) self_variables = (inputs=inputs_(m.value), outputs=outputs_(m.value)) # hard_dep_vars = map(variables, m.hard_dependencies) diff --git a/src/run.jl b/src/run.jl index 53696add6..98362eeeb 100644 --- a/src/run.jl +++ b/src/run.jl @@ -74,52 +74,78 @@ julia> meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0); Run the simulation: ```jldoctest run -julia> run!(models, meteo); +julia> outputs_sim = run!(models, meteo); ``` Get the results: ```jldoctest run -julia> (models[:var4],models[:var6]) +julia> (outputs_sim[:var4],outputs_sim[:var6]) ([12.0], [41.95]) ``` """ run! -# Managing one or several objects, one or several time-steps: -# This is the default function called by the user, which uses traits -# to dispatch to the correct method. The traits are defined in table_traits.jl -# and define either TableAlike or SingletonAlike objects. -# Please use these traits to define your own objects. + +function adjust_weather_timesteps_to_given_length(desired_length, meteo) + # This isn't ideal in terms of codeflow, but check_dimensions will kick in later + # And determine whether there is a status vector length discrepancy + + meteo_adjusted = meteo + + if DataFormat(meteo_adjusted) == TableAlike() + return Tables.rows(meteo_adjusted) + end + + if isnothing(meteo) + meteo_adjusted = Weather(repeat([Atmosphere(NamedTuple())], desired_length)) + elseif get_nsteps(meteo) == 1 && desired_length > 1 + if isa(meteo, Atmosphere) + meteo_adjusted = Weather(repeat([meteo], desired_length)) + end + end + + return meteo_adjusted +end + + +# User entry point, which uses traits to dispatch to the correct method. +# The traits are defined in table_traits.jl +# and define either TableAlike, TreeAlike or SingletonAlike objects. function run!( object, meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; + tracked_outputs=nothing, check=true, executor=ThreadedEx() ) run!( DataFormat(object), - DataFormat(meteo), object, meteo, constants, extra; + tracked_outputs, check, executor ) end -# 1- several objects and several time-steps +########################################################################################## +## ModelList (single-scale) simulations +########################################################################################## + +# 1- several ModelList objects and several time-steps function run!( - ::TableAlike, ::TableAlike, object::T, meteo::TimeStepTable{A}, constants=PlantMeteo.Constants(), extra=nothing; + tracked_outputs=nothing, check=true, executor=ThreadedEx() ) where {T<:Union{AbstractArray,AbstractDict},A} @@ -130,93 +156,42 @@ function run!( ) maxlog = 1 end - for obj in collect(values(object)) - run!(obj, meteo, constants, extra, check=check, executor=executor) - end -end + outputs_collection = isa(object, AbstractArray) ? [] : isnothing(tracked_outputs) ? Dict() : Dict{TimeStepTable{Status{typeof(tracked_outputs)}}} + # Each object: + for obj in object -# 2- one object, one time-step -function run!( - ::SingletonAlike, - ::SingletonAlike, - object, - meteo=nothing, - constants=PlantMeteo.Constants(), - extra=nothing; - check=true -) - run!(object, Weather[meteo], constants, extra; check) + if isa(object, AbstractArray) + push!(outputs_collection, run!(obj, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor)) + else + outputs_collection[obj.first] = run!(obj.second, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor) + end + + end + return outputs_collection end -# 3- one object, one meteo time-step, several status time-steps (rare case but possible) -# Also occurs when meteo is nothing +# 2 - One object, one or multiple meteo time-step(s), with vectors provided in the status +# (meaning a single meteo timestep might be expanded to fit the status vector size) function run!( - ::TableAlike, ::SingletonAlike, object::T, meteo=nothing, constants=PlantMeteo.Constants(), extra=nothing; + tracked_outputs=nothing, check=true, executor=ThreadedEx() ) where {T<:ModelList} - sim_rows = Tables.rows(status(object)) - dep_graph = dep(object, length(sim_rows)) - - if check && length(dep_graph.not_found) > 0 - error( - "The following processes are missing to run the ModelList: ", - dep_graph.not_found - ) - end - - if !timestep_parallelizable(dep_graph) - if executor != SequentialEx() - is_ts_parallel = which_timestep_parallelizable(dep_graph) - mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") + + meteo_adjusted = adjust_weather_timesteps_to_given_length(get_status_vector_max_length(object.status), meteo) + nsteps = get_nsteps(meteo_adjusted) - check && @warn string( - "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", - "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." - ) maxlog = 1 - end - # Not parallelizable over time-steps, it means some values depend on the previous value. - # In this case we propagate the values of the variables from one time-step to the other, except for - # the variables the user provided for all time-steps. - for (i, row) in enumerate(sim_rows) - i > 1 && propagate_values!(sim_rows[i-1], row, object.vars_not_propagated) - roots = collect(dep_graph.roots) - for (process, node) in roots - run_node!(object, node, i, row, meteo, constants, extra) - end end - else - @floop executor for (i, row) in enumerate(sim_rows) - local roots = collect(dep_graph.roots) - for (process, node) in roots - run_node!(object, node, i, row, meteo, constants, extra) - end - end - end -end - -# 4- one object, several meteo time-step, several status time-steps -function run!( - ::TableAlike, - ::TableAlike, - object::T, - meteo, - constants=PlantMeteo.Constants(), - extra=nothing; - check=true, - executor=ThreadedEx() -) where {T<:ModelList} - meteo_rows = Tables.rows(meteo) - dep_graph = dep(object, length(meteo_rows)) + dep_graph = dep(object, nsteps) if check # Check if the meteo data and the status have the same length (or length 1) - check_dimensions(object, meteo) + check_dimensions(object, meteo_adjusted) if length(dep_graph.not_found) > 0 error( @@ -226,8 +201,9 @@ function run!( end end - if !timestep_parallelizable(dep_graph) - if executor != SequentialEx() + + if executor != SequentialEx() && nsteps > 1 + if !timestep_parallelizable(dep_graph) is_ts_parallel = which_timestep_parallelizable(dep_graph) mods_not_parallel = join([i.second.first for i in is_ts_parallel[findall(x -> x.second.second == false, is_ts_parallel)]], "; ") @@ -235,44 +211,72 @@ function run!( "A parallel executor was provided (`executor=$(executor)`) but some models cannot be run in parallel: $mods_not_parallel. ", "The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." ) maxlog = 1 + else + outputs_preallocated_mt = pre_allocate_outputs(object, tracked_outputs, nsteps) + local vars = length(outputs_preallocated_mt) > 0 ? keys(outputs_preallocated_mt[1]) : NamedTuple() + status_flattened_template, vector_variables_mt = flatten_status(object.status) + + # Computing time-steps in parallel: + @floop executor for i in 1:nsteps + @init begin + status_flattened = deepcopy(status_flattened_template) + roots = collect(dep_graph.roots) + end + meteo_i = meteo_adjusted[i] + update_vector_variables(object.status, status_flattened, vector_variables_mt, i) + for (process, node) in roots + run_node!(object, node, i, status_flattened, meteo_i, constants, extra) + end + for var in vars + outputs_preallocated_mt[i][var] = status_flattened[var] + end + end + return outputs_preallocated_mt end + end - # Not parallelizable over time-steps, it means some values depend on the previous value. - # In this case we propagate the values of the variables from one time-step to the other, except for - # the variables the user provided for all time-steps. - roots = collect(dep_graph.roots) + outputs_preallocated = pre_allocate_outputs(object, tracked_outputs, nsteps) + status_flattened, vector_variables = flatten_status(object.status) - for (i, meteo_i) in enumerate(meteo_rows) - i > 1 && propagate_values!(object[i-1], object[i], object.vars_not_propagated) - for (process, node) in roots - run_node!(object, node, i, object[i], meteo_i, constants, extra) - end - end + # Not parallelizable over time-steps, it means some values depend on the previous value. + # In this case we propagate the values of the variables from one time-step to the other, except for + # the variables the user provided for all time-steps. + roots = collect(dep_graph.roots) + + # this bit is necessary for DataFrameRow meteos, see XPalm tests + if nsteps == 1 + for (process, node) in roots + run_node!(object, node, 1, status_flattened, meteo_adjusted, constants, extra) + end + save_results!(status_flattened, outputs_preallocated, 1) else - # Computing time-steps in parallel: - @floop executor for (i, meteo_i) in enumerate(meteo_rows) - local roots = collect(dep_graph.roots) + + for (i, meteo_i) in enumerate(meteo_adjusted) for (process, node) in roots - run_node!(object, node, i, object[i], meteo_i, constants, extra) + run_node!(object, node, i, status_flattened, meteo_i, constants, extra) end + save_results!(status_flattened, outputs_preallocated, i) + i + 1 <= nsteps && update_vector_variables(object.status, status_flattened, vector_variables, i + 1) end end + + return outputs_preallocated end -# 5- several objects and one meteo time-step +# 3- several objects and one meteo time-step function run!( ::TableAlike, - ::SingletonAlike, object::T, meteo, constants=PlantMeteo.Constants(), extra=nothing; + tracked_outputs=nothing, check=true, executor=ThreadedEx() -) where {T<:Union{AbstractArray,AbstractDict}} +) where {T<:Union{AbstractArray, AbstractDict}} dep_graphs = [dep(obj) for obj in collect(values(object))] - obj_parallelizable = all([object_parallelizable(graph) for graph in dep_graphs]) + #obj_parallelizable = all([object_parallelizable(graph) for graph in dep_graphs]) # Check if the simulation can be parallelized over objects: if executor != SequentialEx() @@ -280,9 +284,10 @@ function run!( "Parallelisation over objects was removed, (but may be reintroduced in the future). Parallelisation will only occur over timesteps." ) maxlog = 1 end + # Each object: for (i, obj) in enumerate(collect(values(object))) - + if check # Check if the meteo data and the status have the same length (or length 1) check_dimensions(obj, meteo) @@ -294,17 +299,26 @@ function run!( ) end end + end + + outputs_collection = isa(object, AbstractArray) ? [] : isnothing(tracked_outputs) ? Dict() : Dict{TimeStepTable{Status{typeof(tracked_outputs)}}} - roots_i = collect(dep_graphs[i].roots) - for (process_i, node_i) in roots_i - run_node!(obj, node_i, 1, status(obj)[1], meteo, constants, extra) + # Each object: + for obj in object + if isa(object, AbstractArray) + push!(outputs_collection, run!(obj, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor)) + else + outputs_collection[obj.first] = run!(obj.second, meteo, constants, extra, tracked_outputs=tracked_outputs, check=check, executor=executor) end + end + return outputs_collection end -# for each dependency node in the graph (always one time-step, one object), actual workhorse: +# Not exposed to the user : +# for each dependency node in the graph (always one time-step, one object), actual workhorse function run_node!( object::T, node::SoftDependencyNode, @@ -335,10 +349,13 @@ function run_node!( end -# Compatibility with MTG: +########################################################################################## +### Multiscale simulations +########################################################################################## +# Another user entry point # If we pass an MTG and a mapping, then we use them to compute a GraphSimulation object -# that we use with the first method in this file. +# that we then use with the generic run! entry point. function run!( object::MultiScaleTreeGraph.Node, mapping::Dict{String,T} where {T}, @@ -346,45 +363,35 @@ function run!( constants=PlantMeteo.Constants(), extra=nothing; nsteps=nothing, - outputs=nothing, + tracked_outputs=nothing, check=true, executor=ThreadedEx() ) isnothing(nsteps) && (nsteps = get_nsteps(meteo)) + meteo_adjusted = adjust_weather_timesteps_to_given_length(nsteps, meteo) - sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=outputs) + # NOTE : replace_mapping_status_vectors_with_generated_models is assumed to have already run if used + # otherwise there might be vector length conflicts with timesteps + sim = GraphSimulation(object, mapping, nsteps=nsteps, check=check, outputs=tracked_outputs) run!( sim, - meteo, + meteo_adjusted, constants, extra; check=check, executor=executor ) - return sim + return outputs(sim) end function run!( ::TreeAlike, - ::SingletonAlike, - object::GraphSimulation, - meteo, - constants=PlantMeteo.Constants(), - extra=nothing; - check=true, - executor=ThreadedEx() -) - run!(object, Weather[meteo], constants, extra, check, executor) -end - -function run!( - ::TreeAlike, - ::TableAlike, object::GraphSimulation, meteo, constants=PlantMeteo.Constants(), extra=nothing; + tracked_outputs=nothing, check=true, executor=ThreadedEx() ) @@ -395,19 +402,31 @@ function run!( !isnothing(extra) && error("Extra parameters are not allowed for the simulation of an MTG (already used for statuses).") - for (i, meteo_i) in enumerate(Tables.rows(meteo)) - roots = collect(dep_graph.roots) + nsteps = get_nsteps(meteo) + + # if this function is called directly with an atmosphere, don't use the Rows interface + if nsteps == 1 + roots = collect(dep_graph.roots) for (process_key, dependency_node) in roots - # Note: parallelization over objects is handled by the run! method below - run_node_multiscale!(object, dependency_node, i, models, meteo_i, constants, object, check, executor) + run_node_multiscale!(object, dependency_node, 1, models, meteo, constants, object, check, executor) + end + save_results!(object, 1) + else + for (i, meteo_i) in enumerate(Tables.rows(meteo)) + roots = collect(dep_graph.roots) + for (process_key, dependency_node) in roots + run_node_multiscale!(object, dependency_node, i, models, meteo_i, constants, object, check, executor) + end + # At the end of the time-step, we save the results of the simulation in the object: + save_results!(object, i) end - # At the end of the time-step, we save the results of the simulation in the object: - save_results!(object, i) end + + return outputs(object) end -# For a tree-alike object: +# Function that runs on dependency graph nodes, actual workhorse : function run_node_multiscale!( object::T, node::SoftDependencyNode, @@ -430,16 +449,6 @@ function run_node_multiscale!( node_statuses = status(object)[node.scale] # Get the status of the nodes at the current scale models_at_scale = models[node.scale] - # Check if the simulation can be parallelized over objects: - #TODO: move this check up in the call stack so we check only once per time-step - if !last(object_parallelizable(node)) && executor != SequentialEx() - check && @warn string( - "A parallel executor was provided (`executor=$(executor)`) but the model $(node.value) (or its hard dependencies) cannot be run in parallel over objects.", - " The simulation will be run sequentially. Use `executor=SequentialEx()` to remove this warning." - ) maxlog = 1 - executor = SequentialEx() - end - for st in node_statuses # for each node status at the current scale (potentially in parallel over nodes) # Actual call to the model: run!(node.value, models_at_scale, st, meteo, constants, extra) diff --git a/src/traits/table_traits.jl b/src/traits/table_traits.jl index 39c1c98e6..2e6373273 100644 --- a/src/traits/table_traits.jl +++ b/src/traits/table_traits.jl @@ -56,8 +56,7 @@ DataFormat(::Type{<:Dict}) = TableAlike() DataFormat(::Type{<:NamedTuple}) = SingletonAlike() DataFormat(::Type{<:Status}) = SingletonAlike() -DataFormat(::Type{<:ModelList{Mo,S,V} where {Mo,S<:Status,V}}) = SingletonAlike() -DataFormat(::Type{<:ModelList{Mo,S,V}}) where {Mo,S,V} = TableAlike() +DataFormat(::Type{<:ModelList{Mo,S} where {Mo,S}}) = SingletonAlike() DataFormat(::Type{<:GraphSimulation}) = TreeAlike() DataFormat(::Type{<:PlantMeteo.AbstractAtmosphere}) = SingletonAlike() diff --git a/test/Project.toml b/test/Project.toml index 6ae7f9b2f..f806effc7 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,10 +1,13 @@ [deps] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantMeteo = "4630fe09-e0fb-4da5-a846-781cb73437b6" +PlantSimEngine = "9a576370-710b-4269-adf9-4f603a9c6423" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/downstream/Project.toml b/test/downstream/Project.toml index f3acf78ca..397e54380 100644 --- a/test/downstream/Project.toml +++ b/test/downstream/Project.toml @@ -2,8 +2,10 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantBiophysics = "7ae8fcfa-76ad-4ec6-9ea7-5f8f5e2d6ec9" PlantMeteo = "4630fe09-e0fb-4da5-a846-781cb73437b6" +PlantSimEngine = "9a576370-710b-4269-adf9-4f603a9c6423" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/downstream/test-PSE-benchmark.jl b/test/downstream/test-PSE-benchmark.jl new file mode 100644 index 000000000..3a7131d3b --- /dev/null +++ b/test/downstream/test-PSE-benchmark.jl @@ -0,0 +1,122 @@ +############################################# +### Simulation with many organs in the MTG (but only a few different types of organs) + + +PlantSimEngine.@process "organ_crazy_emergence" verbose = false + +""" + ToyInternodeCrazyEmergence(;init_TT=0.0, TT_emergence = 300) + +Computes the organ emergence based on cumulated thermal time since last event. +""" +struct ToyInternodeCrazyEmergence <: AbstractOrgan_Crazy_EmergenceModel + TT_emergence::Float64 +end + +ToyInternodeCrazyEmergence(; TT_emergence=300.0) = ToyInternodeCrazyEmergence(TT_emergence) + +PlantSimEngine.inputs_(m::ToyInternodeCrazyEmergence) = (TT_cu=-Inf,) +PlantSimEngine.outputs_(m::ToyInternodeCrazyEmergence) = (TT_cu_emergence=0.0,) + +function PlantSimEngine.run!(m::ToyInternodeCrazyEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + + #root = get_root(status.node) + + #if nleaves(root) > 10000 + # return nothing + #end + + if length(MultiScaleTreeGraph.children(status.node)) == 1 && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) + status_new_internode.TT_cu_emergence = status.TT_cu + elseif (length(MultiScaleTreeGraph.children(status.node)) >= 2 && length(MultiScaleTreeGraph.children(status.node)) < 7) && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=4) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=5) + status_new_internode.TT_cu_emergence = status.TT_cu + elseif (length(MultiScaleTreeGraph.children(status.node)) >= 7 && length(MultiScaleTreeGraph.children(status.node)) < 30) && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=6) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=7) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=8) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=9) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=10) + add_organ!(status.node, sim_object, "+", "Leaf", 2, index=11) + + end + + return nothing +end + + +# Wrapped this into a function so that it doesn't plague the benchmark with variables on a global scope +#@check_allocs +function do_benchmark_on_heavier_mtg() + mtg = import_mtg_example(); + + # Example meteo, 365 timesteps : + meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) + + #similar to the mtg growth test but with a much lower emergence threshold + mapping = Dict( + "Scene" => ToyDegreeDaysCumulModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[ + :TT_cu => "Scene", + ], + ), + PlantSimEngine.Examples.Beer(0.6), + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + ), + ), + "Internode" => ( + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",], + ), + MultiScaleModel( + model=ToyInternodeCrazyEmergence(TT_emergence=1.0), + mapped_variables=[:TT_cu => "Scene"], + ), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(carbon_biomass=1.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil", :aPPFD => "Plant"], + ), + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",], + ), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + Status(carbon_biomass=1.0) + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + + out_vars = Dict( + "Leaf" => (:carbon_assimilation, :carbon_demand, :soil_water_content, :carbon_allocation), + "Internode" => (:carbon_allocation, :TT_cu_emergence), + "Plant" => (:carbon_allocation,), + "Soil" => (:soil_water_content,), + ) + + out = run!(mtg, mapping, meteo_day, tracked_outputs=out_vars, executor=SequentialEx()); +end \ No newline at end of file diff --git a/test/downstream/test-all-benchmarks.jl b/test/downstream/test-all-benchmarks.jl new file mode 100644 index 000000000..36c059f0f --- /dev/null +++ b/test/downstream/test-all-benchmarks.jl @@ -0,0 +1,51 @@ +using Pkg +Pkg.activate(dirname(@__FILE__)) +Pkg.develop(PackageSpec(path=dirname(dirname(@__DIR__)))) +Pkg.instantiate() + +using PlantSimEngine +using PlantSimEngine.Examples +using DataFrames, CSV +using MultiScaleTreeGraph +using PlantMeteo, Statistics + +using BenchmarkTools +using Dates + +suite_name = "bench_" + +if Sys.iswindows() + suite_name = suite_name * "windows" +elseif Sys.isapple() + suite_name = suite_name * "mac" +elseif Sys.islinux() + suite_name = suite_name * "linux" +end +suite = BenchmarkGroup() +suite[suite_name] = BenchmarkGroup(["PSE", "PBP", "XPalm"]) + +# "PSE benchmark" +include("test-PSE-benchmark.jl") +suite[suite_name]["PSE"] = @benchmarkable do_benchmark_on_heavier_mtg() + +# "PBP benchmark" +include("test-plantbiophysics.jl") +suite[suite_name]["PBP"] = @benchmarkable benchmark_plantbiophysics() + +leaf, meteo = setup_benchmark_plantbiophysics_multitimestep() +suite[suite_name]["PBP_multiple_timesteps_MT"] = @benchmarkable benchmark_plantbiophysics_multitimestep_MT($leaf, $meteo) +suite[suite_name]["PBP_multiple_timesteps_ST"] = @benchmarkable benchmark_plantbiophysics_multitimestep_ST($leaf, $meteo) + +# "XPalm benchmark" +#=include("test-xpalm.jl") +suite[suite_name]["XPalm_setup"] = @benchmarkable xpalm_default_param_create() seconds = 120 + +palm, models, out_vars, meteo = xpalm_default_param_create() +sim_outputs = xpalm_default_param_run(palm, models, out_vars, meteo) + +suite[suite_name]["XPalm_run"] = @benchmarkable xpalm_default_param_run($palm, $models, $out_vars, $meteo) seconds = 120 +suite[suite_name]["XPalm_convert_outputs"] = @benchmarkable xpalm_default_param_convert_outputs($sim_outputs) seconds = 120 +=# +tune!(suite) +results = run(suite, verbose=true) +BenchmarkTools.save(dirname(@__FILE__) * "/output.json", median(results)) \ No newline at end of file diff --git a/test/downstream/test-plantbiophysics.jl b/test/downstream/test-plantbiophysics.jl index f8c1dd47c..8f026c30b 100644 --- a/test/downstream/test-plantbiophysics.jl +++ b/test/downstream/test-plantbiophysics.jl @@ -1,15 +1,15 @@ -#TODO Cleanup +# For local testing : #using Pkg #Pkg.develop("PlantSimEngine") #using PlantSimEngine using Statistics -using DataFrames -using CSV +#using DataFrames +#using CSV using Random using PlantBiophysics -using BenchmarkTools -using Test -using PlantMeteo +#using BenchmarkTools +#using Test +#using PlantMeteo function benchmark_plantbiophysics() @@ -60,7 +60,7 @@ function benchmark_plantbiophysics() constants = Constants() - time_PB = Vector{Float64}(undef, N*microbenchmark_steps) + #time_PB = Vector{Float64}(undef, N*microbenchmark_steps) for i = 1:N leaf = ModelList( energy_balance=Monteith(), @@ -78,77 +78,104 @@ function benchmark_plantbiophysics() d=set.d[i], ), ) - deps = PlantSimEngine.dep(leaf) + #deps = PlantSimEngine.dep(leaf) meteo = Atmosphere(T=set.T[i], Wind=set.Wind[i], P=set.P[i], Rh=set.Rh[i], Cₐ=set.Ca[i]) - st = PlantMeteo.row_struct(leaf.status[1]) - b_PB = @benchmark run!($leaf, $deps, 1, $st, $meteo, $constants, nothing; executor = ThreadedEx()) evals = microbenchmark_evals samples = microbenchmark_steps - + #st = PlantMeteo.row_struct(leaf.status[1]) + #b_PB = @benchmark run!($leaf, $meteo, $constants, nothing; executor = ThreadedEx()) evals = microbenchmark_evals samples = microbenchmark_steps + run!(leaf, meteo, constants, nothing; executor = ThreadedEx()) + # transform in seconds - for j in 1:microbenchmark_steps + #=for j in 1:microbenchmark_steps time_PB[microbenchmark_steps*(i-1) + j] = b_PB.times[j]*1e-9 - end + end=# end - return time_PB + #return time_PB end -@testset "PlantBiophysics benchmark" begin - - time_PB = benchmark_plantbiophysics() - N = length(time_PB) - #statsPB = (mean(time_PB), median(time_PB), Statistics.std(time_PB), findmin(time_PB), findmax(time_PB)) - min__ = findmin(time_PB)[1] - max__ = findmin(time_PB)[1] - @test min__ > 1e-7 - @test max__ < 1e-5 - @test mean(time_PB) > 5e-7 - @test mean(time_PB) < 5e-6 - #TODO -end +function setup_benchmark_plantbiophysics_multitimestep() -#= -function run_plantbiophysics() - - - Rs = 10.0 - Ta = 18.0 - Wind = 0.5 - P = 90.0 - Rh = 0.1 - Ca = 360.0 - skyF = 0.0 - d = 0.001 - Jmax = 200.0 - Vmax = 150.0 - Rd = 0.3 - TPU = 5.0 - g0 = 0.001 - g1 = 0.5 - vpd = e_sat(Ta) - vapor_pressure(Ta, Rh) - PPFD = Rs*0.48*4.57 + Random.seed!(1) # Set random seed + N = 100 # Number of timesteps simulated for each microbenchmark step - constants = Constants() + length_range = 10000 + Rs = range(10, 500, length=length_range) + Ta = range(18, 40, length=length_range) + Wind = range(0.5, 20, length=length_range) + P = range(90, 101, length=length_range) + Rh = range(0.1, 0.98, length=length_range) + Ca = range(360, 900, length=length_range) + skyF = range(0.0, 1.0, length=length_range) + d = range(0.001, 0.5, length=length_range) + Jmax = range(200.0, 300.0, length=length_range) + Vmax = range(150.0, 250.0, length=length_range) + Rd = range(0.3, 2.0, length=length_range) + TPU = range(5.0, 20.0, length=length_range) + g0 = range(0.001, 2.0, length=length_range) + g1 = range(0.5, 15.0, length=length_range) + vars = hcat([Ta, Wind, P, Rh, Ca, Jmax, Vmax, Rd, Rs, skyF, d, TPU, g0, g1]) - leaf = ModelList( + set = [rand.(vars) for i = 1:N] + set = reshape(vcat(set...), (length(set[1]), length(set)))' + name = [ + "T", + "Wind", + "P", + "Rh", + "Ca", + "JMaxRef", + "VcMaxRef", + "RdRef", + "Rs", + "sky_fraction", + "d", + "TPURef", + "g0", + "g1", + ] + set = DataFrame(set, name) + @. set[!, :vpd] = e_sat(set.T) - vapor_pressure(set.T, set.Rh) + @. set[!, :PPFD] = set.Rs * 0.48 * 4.57 + + leaf = Vector{ModelList}(undef, N) + for i = 1:N + + leaf[i] = ModelList( energy_balance=Monteith(), photosynthesis=Fvcb( - VcMaxRef=Vmax, - JMaxRef=Jmax, - RdRef=Rd, - TPURef=TPU, + VcMaxRef=set.VcMaxRef[i], + JMaxRef=set.JMaxRef[i], + RdRef=set.RdRef[i], + TPURef=set.TPURef[i], ), - stomatal_conductance=Medlyn(g0, g1), + stomatal_conductance=Medlyn(set.g0[i], set.g1[i]), status=( - Rₛ=Rs, - sky_fraction=skyF, - PPFD=PPFD, - d=d, + Rₛ=set.Rs, + sky_fraction=set.sky_fraction, + PPFD=set.PPFD, + d=set.d, ), ) - deps = PlantSimEngine.dep(leaf) - meteo = Atmosphere(T=Ta, Wind=Wind, P=P, Rh=Rh, Cₐ=Ca) - st = PlantMeteo.row_struct(leaf.status[1]) - run!(leaf, deps, 1, st, meteo, constants, nothing) + end + + atm = Vector{Atmosphere}(undef, N) + for i in 1:N + atm[i]= Atmosphere(T=set.T[i], Wind=set.Wind[i], P=set.P[i], Rh=set.Rh[i], Cₐ=set.Ca[i]) + end + meteo = Weather(atm) + + return leaf, meteo end -run_plantbiophysics() -=# \ No newline at end of file +function benchmark_plantbiophysics_multitimestep_MT(leaf, meteo) + N = length(meteo) + for i in 1:N + run!(leaf[i], meteo, Constants(), nothing; executor = ThreadedEx()) + end +end + +function benchmark_plantbiophysics_multitimestep_ST(leaf, meteo) + N = length(meteo) + for i in 1:N + run!(leaf[i], meteo, Constants(), nothing; executor = SequentialEx()) + end +end \ No newline at end of file diff --git a/test/downstream/test-xpalm.jl b/test/downstream/test-xpalm.jl index 5e199e395..c2708374d 100644 --- a/test/downstream/test-xpalm.jl +++ b/test/downstream/test-xpalm.jl @@ -1,5 +1,59 @@ +#using Pkg +#Pkg.develop("PlantSimEngine") +#using PlantSimEngine + +# no release of XPalm yet, so can't just add it to the .toml +using Pkg +Pkg.add(url="https://github.com/PalmStudio/XPalm.jl") + using Test +using PlantMeteo#, MultiScaleTreeGraph +#using CairoMakie, AlgebraOfGraphics +using DataFrames, CSV, Statistics +using Dates +using XPalm +using BenchmarkTools + +function xpalm_default_param_create() + meteo = CSV.read("../XPalm.jl/0-data/Meteo_Nigeria_PR.txt", DataFrame) + meteo.duration = [Dates.Day(i[1:1]) for i in meteo.duration] + m = Weather(meteo) + + out_vars = Dict{String,Any}( + "Scene" => (:lai,), + # "Scene" => (:lai, :scene_leaf_area, :aPPFD, :TEff), + # "Plant" => (:plant_age, :ftsw, :newPhytomerEmergence, :aPPFD, :plant_leaf_area, :carbon_assimilation, :carbon_offer_after_rm, :Rm, :TT_since_init, :TEff, :phytomer_count, :newPhytomerEmergence), + "Leaf" => (:Rm, :potential_area, :TT_since_init, :TEff, :A, :carbon_demand, :carbon_allocation,), + # "Leaf" => (:Rm, :potential_area), + # "Internode" => (:Rm, :carbon_allocation, :carbon_demand), + "Male" => (:Rm,), + # "Female" => (:biomass,), + # "Soil" => (:TEff, :ftsw, :root_depth), + ) + + # Example 1: Run the model with the default parameters (but output as a DataFrame): + palm = Palm(initiation_age=0, parameters=default_parameters()) + models = model_mapping(palm) + return palm, models, out_vars, meteo +end + +function xpalm_default_param_run(palm, models, meteo, out_vars) + sim_outputs = PlantSimEngine.run!(palm.mtg, models, meteo, tracked_outputs=out_vars, executor=PlantSimEngine.SequentialEx(), check=false) + return sim_outputs +end + +function xpalm_default_param_convert_outputs(sim_outputs) + df = PlantSimEngine.convert_outputs(out, DataFrame, no_value=missing) + return df +end + + +#=@testset "XPalm simple test" begin + # default number of seconds is 5 + b_XP = @benchmark xpalm_default_param_run() seconds = 120 + + #N = length(b_XP.times) -@testset "XPalm dummy test TODO" begin - @test true -end \ No newline at end of file + @test mean(b_XP.times*1e-9) > 10 + @test mean(b_XP.times*1e-9) < 15 +end =# \ No newline at end of file diff --git a/test/helper-functions.jl b/test/helper-functions.jl index ac83bff20..9cf3f8477 100644 --- a/test/helper-functions.jl +++ b/test/helper-functions.jl @@ -1,25 +1,32 @@ # Simple helper functions that can be used in various tests here and there +function compare_outputs_modellist_mapping(filtered_outputs, graphsim) + outputs_df = convert_outputs(graphsim.outputs, DataFrame) -function compare_outputs_modellist_mapping(models, graphsim) - graphsim_df = outputs(graphsim, DataFrame) - - graphsim_df_outputs_only = select(graphsim_df, Not([:timestep, :organ, :node])) - models_df = DataFrame(status(models)) + outputs_df_outputs_only = select(outputs_df, Not([:timestep, :organ, :node])) + models_df = DataFrame(filtered_outputs) models_df_sorted = models_df[:, sortperm(names(models_df))] - graphsim_df_outputs_only_sorted = graphsim_df_outputs_only[:, sortperm(names(graphsim_df_outputs_only))] - return graphsim_df_outputs_only_sorted == models_df_sorted + outputs_df_outputs_only_sorted = outputs_df_outputs_only[:, sortperm(names(outputs_df_outputs_only))] + return outputs_df_outputs_only_sorted == models_df_sorted end # doesn't check for mtg equality function compare_outputs_graphsim(graphsim, graphsim2) - graphsim_df = outputs(graphsim, DataFrame) - graphsim_df_sorted = graphsim_df[:, sortperm(names(graphsim_df))] + outputs_df = convert_outputs(graphsim.outputs, DataFrame) + outputs_df_sorted = outputs_df[:, sortperm(names(outputs_df))] - graphsim2_df = outputs(graphsim2, DataFrame) - graphsim2_df_sorted = graphsim2_df[:, sortperm(names(graphsim2_df))] - return graphsim_df_sorted == graphsim2_df_sorted + outputs2_df = convert_outputs(graphsim2.outputs, DataFrame) + outputs2_df_sorted = outputs2_df[:, sortperm(names(outputs2_df))] + return outputs_df_sorted == outputs2_df_sorted +end + +function compare_outputs_modellists(filtered_outputs_1, filtered_outputs_2) + models_df_1 = DataFrame(filtered_outputs_1) + models_df_sorted_1 = models_df_1[:, sortperm(names(models_df_1))] + models_df_2 = DataFrame(filtered_outputs_2) + models_df_sorted_2 = models_df_2[:, sortperm(names(models_df_2))] + return models_df_sorted_2 == models_df_sorted_1 end # Breaking this function into two to ensure eval() state synchronisation happens (see comments around the modellist_to_mapping definition) @@ -30,8 +37,8 @@ function check_multiscale_simulation_is_equivalent_begin(models::ModelList, stat return mtg, mapping, out end -function check_multiscale_simulation_is_equivalent_end(models::ModelList, mtg, mapping, out, meteo) - graph_sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=length(meteo), check=true, outputs=out) +function check_multiscale_simulation_is_equivalent_end(modellist_outputs, mtg, mapping, out, meteo) + graph_sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=PlantSimEngine.get_nsteps(meteo), check=true, outputs=out) sim = run!(graph_sim, meteo, @@ -41,5 +48,284 @@ function check_multiscale_simulation_is_equivalent_end(models::ModelList, mtg, m executor=SequentialEx() ); - return compare_outputs_modellist_mapping(models, graph_sim) + return compare_outputs_modellist_mapping(modellist_outputs, graph_sim) +end + +# Quick and naive first version. Doesn't check if everything is timestep parallelizable, doesn't check for nthreads etc. +function run_single_and_multi_thread_modellist(modellist, tracked_outputs, meteo) + out_seq = run!(modellist, meteo; tracked_outputs = tracked_outputs, executor = SequentialEx()) + modellist_mt = copy(modellist) + out_mt = run!(modellist_mt, meteo; tracked_outputs = tracked_outputs, executor = ThreadedEx()) + return out_seq, out_mt +end + +# Could make use of PlantMeteo's online meteo data recovery feature for more numerous examples +# or the random meteo generation used for the PBP benchmark + +#=using PlantMeteo, Dates, DataFrames +# Define the period of the simulation: +period = [Dates.Date("2021-01-01"), Dates.Date("2021-12-31")] +# Get the weather data for CIRAD's site in Montpellier, France: +meteo = get_weather(43.649777, 3.869889, period, sink = DataFrame)=# + +function get_simple_meteo_bank() + meteos= + [#=nothing,=# Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0), + Weather( + [ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=300.0), + Atmosphere(T=25.0, Wind=0.5, Rh=0.8, Ri_PAR_f=500.0) + ]), + + Weather([Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=200.0), + Atmosphere(T=18.0, Wind=1.0, Rh=0.65, Ri_PAR_f=100.0), + Atmosphere(T=19.0, Wind=1.0, Rh=0.65, Ri_PAR_f=200.0), + Atmosphere(T=30.0, Wind=0.5, Rh=0.6, Ri_PAR_f=100.0), + Atmosphere(T=20.0, Wind=1.0, Rh=0.6, Ri_PAR_f=200.0), + Atmosphere(T=25.0, Wind=1.0, Rh=0.6, Ri_PAR_f=200.0), + Atmosphere(T=10.0, Wind=0.5, Rh=0.6, Ri_PAR_f=200.0)]), + + CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18), + + ] + return meteos +end + +function get_modellist_bank() + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + + rue = 0.3 + + vals = (var1=15.0, var2=0.3)#, TT_cu=cumsum(meteo_day.TT)) + vals2 = (TT_cu=cumsum(meteo_day.TT),) + vals3 = (var1=15.0, var2=0.3) + vals4 = (var9 =1.0,var0=1.0) + vals5 = (var0=1.0,) + vals6 = (var0=1.0,) + + status_tuples = [vals, vals2, vals3, vals4, vals5, vals6] + + models = [ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + status=vals + ), + ModelList( + ToyLAIModel(), + Beer(0.5), + ToyRUEGrowthModel(rue), + status=vals2, + ), + + ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + process3=Process3Model(), + status=vals3 + ), + + ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + process3=Process3Model(), + process4=Process4Model(), + process5=Process5Model(), + process6=Process6Model(), + # process7=Process7Model(), + status=vals4 + ), + + ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + process3=Process3Model(), + process4=Process4Model(), + process5=Process5Model(), + process6=Process6Model(), + process7=Process7Model(), + status=vals5 + ), + + ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + process3=Process3Model(), + process4=Process4Model(), + process5=Process5Model(), + status=vals6 + ), + + ] + + outputs_tuples_vectors = + [ + # this one has one tuple with a duplicate, and one with a nonexistent variable + [NamedTuple(), (:var1,), #=(:var1, :var1),=# (:var1, :var2), (:var1, :var3), (:var1, :var4, :var5), + #=(:var2, :var7, :var3, :var1),=# (:var1, :var2, :var3, :var4, :var5)], + + [NamedTuple(), (:TT_cu,), (:TT_cu,:LAI) , (:biomass,:LAI), (:TT_cu, :LAI, :aPPFD, :biomass, :biomass_increment),], + + [NamedTuple(), (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + #=(:var2, :var7, :var3, :var1),=# (:var1, :var2, :var3, :var4, :var5, :var6)], + + [NamedTuple(), (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6)], + + [NamedTuple(), (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6) + , (:var1, :var2, :var3, :var4, :var5, :var6, :var7, :var8, :var9)], + + [NamedTuple(), (:var1,), #=(:var1, :var1),=# (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6) + , #=(:var1, :var2, :var3, :var4, :var5, :var6, :var7, :var8, :var9, :var0)=#], + + ] + + return models, status_tuples, outputs_tuples_vectors +end + +# Could add some mtg variation too +function get_simple_mapping_bank() + mappings = [ + Dict( + "Scene" => ToyDegreeDaysCumulModel(), + "Plant" => ( + MultiScaleModel( + model=ToyLAIModel(), + mapped_variables=[:TT_cu => "Scene",],), + Beer(0.6), + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + :carbon_allocation => ["Leaf", "Internode"]],), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],],),), + "Internode" => ( + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",],), + MultiScaleModel( + model=ToyInternodeEmergence(TT_emergence=20.0), + mapped_variables=[:TT_cu => "Scene"],), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(carbon_biomass=1.0)), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil", :aPPFD => "Plant"],), + MultiScaleModel( + model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + mapped_variables=[:TT => "Scene",],), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + Status(carbon_biomass=1.0)), + "Soil" => (ToySoilWaterModel(),),), +########## + Dict( + "Default" => ( + Process1Model(1.0), + Status(var1=15.0, var2=0.3,),),), +########## + Dict( + "Plant" => ( + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + # inputs + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"]],), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],],),), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(TT=10.0, carbon_biomass=1.0)), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil",], + # Notice we provide "Soil", not ["Soil"], so a single value is expected here + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + Status(aPPFD=1300.0, TT=10.0, carbon_biomass=1.0), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025),), + "Soil" => (ToySoilWaterModel(),),), +################## + ] + +out_vars_vectors = [ + [nothing, + NamedTuple(), + Dict(), + #Dict("Leaf" => NamedTuple()), # incorrect + Dict("Leaf" => (:carbon_allocation,),), + Dict("Leaf" => (:carbon_demand,),), + Dict( + "Leaf" => (:carbon_assimilation, :carbon_demand, :soil_water_content, :carbon_allocation), + "Internode" => (:carbon_allocation, :TT_cu_emergence), + "Plant" => (:carbon_allocation,), + "Soil" => (:soil_water_content,),),], + ############# + [nothing, + NamedTuple(), + Dict("Default" => (:var1,)) + ], + ############# + [ + nothing, + NamedTuple(), + Dict( + "Leaf" => (:carbon_assimilation, :carbon_demand), + "Soil" => (:soil_water_content,), + ),], +] + mtgs = [ + import_mtg_example(), + MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 0, 0 ),), + import_mtg_example() + ] + return mtgs, mappings, out_vars_vectors +end + + +# Split into two parts to ensure eval() syncs and that automatic generation becomes visible for later simulation +# See world-age problems and comments around modellist_to_mapping if you don't know/remember what that's about +function test_filtered_output_begin(m::ModelList, status_tuple, requested_outputs, meteo) + + nsteps = PlantSimEngine.get_nsteps(meteo) + preallocated_outputs = PlantSimEngine.pre_allocate_outputs(m, requested_outputs, nsteps) + @test length(preallocated_outputs) == nsteps + if length(requested_outputs) > 0 + @test length(preallocated_outputs[1]) == length(requested_outputs) + else + # don't compare with the status because unnecessary variables in the status are discarded in the filtered outputs + out_vars_all = merge(init_variables(m; verbose=false)...) + println(out_vars_all) + @test length(preallocated_outputs[1]) == length(out_vars_all) + end + + filtered_outputs_modellist = run!(m, meteo; tracked_outputs=requested_outputs, executor = SequentialEx()) + + # compare filtered output of a modellist with the filtered output of the equivalent simulation in multiscale mode + mtg, mapping, outputs_mapping = PlantSimEngine.modellist_to_mapping(m, status_tuple; nsteps=nsteps, outputs=requested_outputs) + + return mtg, mapping, outputs_mapping, nsteps, filtered_outputs_modellist +end + +function test_filtered_output(mtg, mapping, nsteps, outputs_mapping, meteo, filtered_outputs_modellist) + graphsim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=outputs_mapping) + + sim2 = run!(graphsim, + meteo, + PlantMeteo.Constants(), + nothing; + check=true, + executor=SequentialEx() + ) + return compare_outputs_modellist_mapping(filtered_outputs_modellist, graphsim) end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 5d443737b..6cf550f70 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,6 +9,11 @@ using Documenter # for doctests include("helper-functions.jl") +# There are 3 kinds of tests : +# PSE functionality/feature tests +# Integration tests (launched in Github Actions, they run PBP and XPalm tests) +# Benchmarks both internal and downstream, located in the downstream folder, and run in another Github Action + @testset "Testing PlantSimEngine" begin Aqua.test_all(PlantSimEngine, ambiguities=false) Aqua.test_ambiguities([PlantSimEngine]) @@ -63,6 +68,10 @@ include("helper-functions.jl") include("test-corner-cases.jl") end + @testset "Multithreading" begin + include("test-performance.jl") + end + if VERSION >= v"1.10" # Some formating changed in Julia 1.10, e.g. @NamedTuple instead of NamedTuple. @testset "Doctests" begin diff --git a/test/test-ModelList.jl b/test/test-ModelList.jl index e69527314..b21df0bef 100644 --- a/test/test-ModelList.jl +++ b/test/test-ModelList.jl @@ -1,5 +1,6 @@ # Tests: # Defining a list of models without status: + @testset "ModelList with no status" begin leaf = ModelList( process1=Process1Model(1.0), @@ -11,7 +12,7 @@ @test all(getproperty(leaf.status, i)[1] == getproperty(st, i) for i in keys(st)) @test !is_initialized(leaf) @test to_initialize(leaf) == (process1=(:var1, :var2), process2=(:var1,)) - @test length(status(leaf)) == 1 + @test length(status(leaf)) == 5 # Requiring 3 steps for initialization: leaf = ModelList( @@ -20,8 +21,8 @@ nsteps=3 ) - @test length(status(leaf)) == 3 - @test status(leaf, :var1) == [-Inf, -Inf, -Inf] + @test length(status(leaf)) == 5 + @test status(leaf, :var1) == -Inf end; @@ -83,8 +84,8 @@ end; nsteps=3 ) - @test length(status(leaf)) == 3 - @test status(leaf, :var1) == [15.0, 15.0, 15.0] + @test length(status(leaf)) == 5 + @test status(leaf, :var1) == 15.0 end; @testset "ModelList with fully initialized status" begin @@ -139,13 +140,13 @@ end; # Copy the model list: ml2 = copy(models) - @test DataFrame(status(ml2)) == DataFrame(status(models)) + @test DataFrame(TimeStepTable([status(ml2)])) == DataFrame(TimeStepTable([status(models)])) # Copy the model list with new status: - tst = TimeStepTable([Status(var1=20.0, var2=0.5)]) - ml3 = copy(models, tst) + st = Status(var1=20.0, var2=0.5) + ml3 = copy(models, st) - @test status(ml3) == tst + @test status(ml3) == st @test ml3.models == models.models @@ -205,4 +206,116 @@ end @test process3.children[1].value == Process5Model() @test isa(process3.children[1], PlantSimEngine.SoftDependencyNode) +end + + + + +# very naive function, doesn't generate full partition sets +# insert_errors : could duplicate a value, add a nonexistent one, make one the wrong type ? +#=function generate_output_tuple(vars_tuple, insert_errors, count) + + outputs_tuples_vector = [NamedTuple()] + + # number not exact, but trying every permutation sounds like a waste of time + for i in 1:max(count, length(vars_tuple)) + new_tuple = () + # TODO + new_tuple = (new_tuple..., new_var) + end + return outputs_tuples_vector +end=# + + + +@testset "ModelList outputs preallocation" begin + meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) + vals = (var1=15.0, var2=0.3, TT_cu=cumsum(meteo_day.TT)) + leaf = ModelList( + process1=Process1Model(1.0), + process2=Process2Model(), + status=vals + ) + outs=(:var3,) + + mtg, mapping, outputs_mapping, nsteps, filtered_outputs_modellist = test_filtered_output_begin(leaf, vals, outs, meteo_day) + @test test_filtered_output(mtg, mapping, nsteps, outputs_mapping, meteo_day, filtered_outputs_modellist) + + meteos = + [Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0), + CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18), + ] + modellists, status_tuples, outs_vectors = get_modellist_bank() + + # remove some of the currently unhandled cases + outs_vectors = + [ + # this one has one tuple with a duplicate, and one with a nonexistent variable + [(:var1,), #=(:var1, :var1),=# (:var1, :var2), (:var1, :var3), (:var1, :var4, :var5), + #=(:var2, :var7, :var3, :var1),=# (:var1, :var2, :var3, :var4, :var5)], + [#=NamedTuple(),=# (:TT_cu,), (:TT_cu,:LAI) , (:biomass,:LAI), (:TT_cu, :LAI, :aPPFD, :biomass, :biomass_increment),], + [#=NamedTuple(),=# (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + #=(:var2, :var7, :var3, :var1),=# (:var1, :var2, :var3, :var4, :var5, :var6)], + [#=NamedTuple(),=# (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6)], + [#=NamedTuple(),=# (:var1,), (:var1, :var4), (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6) + , (:var1, :var2, :var3, :var4, :var5, :var6, :var7, :var8, :var9)], + [#=NamedTuple(),=# (:var1,), #=(:var1, :var1),=# (:var1, :var2), (:var1, :var3), (:var1, :var4, :var6, :var5), + (:var2, :var7, :var3, :var1), (:var1, :var2, :var3, :var4, :var5, :var6) + , (:var1, :var2, :var3, :var4, :var5, :var6, :var7, #=:var8, :var9,=# :var0)], + ] + + + + for i in 1:length(modellists) + + modellist = modellists[i] + status_tuple = status_tuples[i] + outs_vector = outs_vectors[i] + all_vars = init_variables(modellist) + + #insert_errors = true + #outs_vector = generate_output_tuple(all_vars, insert_errors) + + for j in 1:length(meteos) + meteo = meteos[j] + for k in 1:length(outs_vector) + out_tuple = outs_vector[k] + #print(i, " ", j, " ", k) + meteo_adjusted = PlantSimEngine.adjust_weather_timesteps_to_given_length( + PlantSimEngine.get_status_vector_max_length(modellist.status) , meteo) + mtg, mapping, outputs_mapping, nsteps, filtered_outputs_modellist = test_filtered_output_begin(modellist, status_tuple, out_tuple, meteo_adjusted) + @test to_initialize(mapping) == Dict() + @test test_filtered_output(mtg, mapping, nsteps, outputs_mapping, meteo_adjusted, filtered_outputs_modellist) + end + end + end + + #mtg, mapping, outputs_mapping, nsteps, filtered_outputs_modellist = test_filtered_output_begin(modellists[1], status_tuples[1], outs_vectors[1][1], meteos[1]) + #@test test_filtered_output(mtg, mapping, nsteps, outputs_mapping, meteo_day, filtered_outputs_modellist) +end + + +PlantSimEngine.@process "modellist_cycle" verbose = false + +struct Reeb{T} <: AbstractModellist_CycleModel + k::T +end + +function PlantSimEngine.run!(::Reeb, models, status, meteo, constants, extra=nothing) + status.LAI = + status.aPPFD + 0.4*k +end + +function PlantSimEngine.inputs_(::Reeb) + (aPPFD=-Inf,) +end + +function PlantSimEngine.outputs_(::Reeb) + (LAI=-Inf,) +end + +@testset "ModelList simple cyclic dependency detection" begin + @test_throws "Cyclic" m = ModelList(Beer(0.5), Reeb(0.5)) end \ No newline at end of file diff --git a/test/test-MultiScaleModel.jl b/test/test-MultiScaleModel.jl index ca02a07dd..673dac327 100644 --- a/test/test-MultiScaleModel.jl +++ b/test/test-MultiScaleModel.jl @@ -18,65 +18,65 @@ end; @testset "MultiScaleModel: case 1" begin models = MultiScaleModel( model=ToyLAIModel(), - mapping=[:TT_cu => "Scene",], + mapped_variables=[:TT_cu => "Scene",], ) @test models.model == ToyLAIModel() - @test models.mapping == [:TT_cu => "Scene" => :TT_cu] + @test models.mapped_variables == [:TT_cu => "Scene" => :TT_cu] end; @testset "MultiScaleModel: case 2" begin models = MultiScaleModel( model=ToyLAIModel(), - mapping=[:TT_cu => ["Plant"],], + mapped_variables=[:TT_cu => ["Plant"],], ) @test models.model == ToyLAIModel() - @test models.mapping == [:TT_cu => ["Plant" => :TT_cu]] + @test models.mapped_variables == [:TT_cu => ["Plant" => :TT_cu]] models = MultiScaleModel( model=ToyLAIModel(), - mapping=[:TT_cu => ["Leaf", "Internode"],], + mapped_variables=[:TT_cu => ["Leaf", "Internode"],], ) @test models.model == ToyLAIModel() - @test models.mapping == [:TT_cu => ["Leaf" => :TT_cu, "Internode" => :TT_cu]] + @test models.mapped_variables == [:TT_cu => ["Leaf" => :TT_cu, "Internode" => :TT_cu]] end; @testset "MultiScaleModel: case 2, several variables with different format" begin models = MultiScaleModel( model=ToyCAllocationModel(), - mapping=[:carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], :Rm => "Plant" => :Rm_plant], + mapped_variables=[:carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], :Rm => "Plant" => :Rm_plant], ) @test models.model == ToyCAllocationModel() - @test models.mapping == [:carbon_assimilation => ["Leaf" => :carbon_assimilation], :carbon_demand => ["Leaf" => :carbon_demand, "Internode" => :carbon_demand], :Rm => "Plant" => :Rm_plant] + @test models.mapped_variables == [:carbon_assimilation => ["Leaf" => :carbon_assimilation], :carbon_demand => ["Leaf" => :carbon_demand, "Internode" => :carbon_demand], :Rm => "Plant" => :Rm_plant] end; @testset "MultiScaleModel: case with PreviousTimeStep => ..." begin models = MultiScaleModel( model=ToyLAIfromLeafAreaModel(1.0), - mapping=[ + mapped_variables=[ PreviousTimeStep(:plant_surfaces) => "Plant" => :surface, ], ) @test models.model == ToyLAIfromLeafAreaModel(1.0) - @test models.mapping == [PreviousTimeStep(:plant_surfaces, :LAI_Dynamic) => ("Plant" => :surface)] + @test models.mapped_variables == [PreviousTimeStep(:plant_surfaces, :LAI_Dynamic) => ("Plant" => :surface)] end; @testset "MultiScaleModel: several types of mapping" begin models = MultiScaleModel( model=ToyLightPartitioningModel(), - mapping=[ + mapped_variables=[ :aPPFD_larger_scale => "Scene" => :aPPFD, :total_surface => "Scene" ], ) @test models.model == ToyLightPartitioningModel() - @test models.mapping == [:aPPFD_larger_scale => ("Scene" => :aPPFD), :total_surface => ("Scene" => :total_surface)] + @test models.mapped_variables == [:aPPFD_larger_scale => ("Scene" => :aPPFD), :total_surface => ("Scene" => :total_surface)] end \ No newline at end of file diff --git a/test/test-Status.jl b/test/test-Status.jl index 96739f8f2..a121d4493 100644 --- a/test/test-Status.jl +++ b/test/test-Status.jl @@ -39,36 +39,34 @@ end status=(var1=[15.0, 16.0], var2=0.3) ) - @test typeof(status(models)) == TimeStepTable{ - Status{ + @test typeof(status(models)) == Status{ (:var5, :var4, :var6, :var1, :var3, :var2), - NTuple{6,Base.RefValue{Float64}} - } - } + Tuple{ + Base.RefValue{Float64}, Base.RefValue{Float64}, Base.RefValue{Float64}, + Base.RefValue{Vector{Float64}}, Base.RefValue{Float64}, Base.RefValue{Float64}} + } + + @test status(models) == models.status @test status(models)[1] == status(models, 1) - @test typeof(status(models, 1)) == PlantMeteo.TimeStepRow{ - Status{ - (:var5, :var4, :var6, :var1, :var3, :var2), - NTuple{6,Base.RefValue{Float64}} - } - } - - @test status(models, 1).var1 == 15.0 - @test status(models, 1).var2 == 0.3 - @test status(models).var1 == [15.0, 16.0] - @test status(models).var2 == [0.3, 0.3] + @test typeof(status(models, 1)) == Float64 + @test typeof(status(models, 4)) == Vector{Float64} + + @test status(models, :var1)[1] == 15.0 + @test status(models, 6) == 0.3 + @test status(models, :var1) == [15.0, 16.0] + @test status(models, :var2) == 0.3 - @test status(models, :var4) == [-Inf, -Inf] - @test status(models, 1).var3 == -Inf - @test status(models, 1).var4 == -Inf - @test status(models, 1).var5 == -Inf - @test status(models, 1).var6 == -Inf + @test status(models, :var4) == -Inf + @test status(models, :var3) == -Inf + @test status(models, :var5) == -Inf + @test status(models, :var6) == -Inf + # TODO this behaviour is now changed, ramifications hard to gauge # Testing setindex: - models[:var6] = [5.5, 5.8] - @test status(models, :var6) == [5.5, 5.8] + #@test models[:var6] = [5.5, 5.8] + #@test status(models, :var6) == [5.5, 5.8] # Testing a vector of ModelList: @test status([models, models]) == [models.status, models.status] diff --git a/test/test-corner-cases.jl b/test/test-corner-cases.jl index b77b02c94..cc881a23d 100644 --- a/test/test-corner-cases.jl +++ b/test/test-corner-cases.jl @@ -10,13 +10,13 @@ # relates to #77 and #99 -PlantSimEngine.@process "Msg3Lvl_amont" verbose = false -PlantSimEngine.@process "Msg3Lvl_amont2" verbose = false -PlantSimEngine.@process "Msg3Lvl_echelle1" verbose = false -PlantSimEngine.@process "Msg3Lvl_echelle2" verbose = false -PlantSimEngine.@process "Msg3Lvl_echelle3" verbose = false -PlantSimEngine.@process "Msg3Lvl_aval" verbose = false -PlantSimEngine.@process "Msg3Lvl_aval2" verbose = false +PlantSimEngine.@process "Msg3Lvl_amont" verbose = false +PlantSimEngine.@process "Msg3Lvl_amont2" verbose = false +PlantSimEngine.@process "Msg3Lvl_echelle1" verbose = false +PlantSimEngine.@process "Msg3Lvl_echelle2" verbose = false +PlantSimEngine.@process "Msg3Lvl_echelle3" verbose = false +PlantSimEngine.@process "Msg3Lvl_aval" verbose = false +PlantSimEngine.@process "Msg3Lvl_aval2" verbose = false # Roots : amont and amont2 # amont2 points to aval @@ -32,11 +32,11 @@ struct Msg3LvlScaleAmontModel <: AbstractMsg3Lvl_AmontModel end function PlantSimEngine.inputs_(::Msg3LvlScaleAmontModel) - (a = -Inf,) + (a=-Inf,) end function PlantSimEngine.outputs_(::Msg3LvlScaleAmontModel) - (b = -Inf, c = -Inf) + (b=-Inf, c=-Inf) end function PlantSimEngine.run!(::Msg3LvlScaleAmontModel, models, status, meteo, constants=nothing, extra_args=nothing) @@ -50,11 +50,11 @@ struct Msg3LvlScaleAmont2Model <: AbstractMsg3Lvl_Amont2Model end function PlantSimEngine.inputs_(::Msg3LvlScaleAmont2Model) - (a2 = -Inf,) + (a2=-Inf,) end function PlantSimEngine.outputs_(::Msg3LvlScaleAmont2Model) - (b2 = -Inf,) + (b2=-Inf,) end function PlantSimEngine.run!(::Msg3LvlScaleAmont2Model, models, status, meteo, constants=nothing, extra_args=nothing) @@ -68,11 +68,11 @@ end function PlantSimEngine.inputs_(::Msg3LvlScaleEchelle3Model) #(b = -Inf, - (c = -Inf,) + (c=-Inf,) end function PlantSimEngine.outputs_(::Msg3LvlScaleEchelle3Model) - (e3 = -Inf, f3 = -Inf) + (e3=-Inf, f3=-Inf) end function PlantSimEngine.run!(::Msg3LvlScaleEchelle3Model, models, status, meteo, constants=nothing, extra_args=nothing) @@ -87,11 +87,11 @@ end function PlantSimEngine.inputs_(::Msg3LvlScaleEchelle2Model) - (c = -Inf, e3 = -Inf, f3 = -Inf) + (c=-Inf, e3=-Inf, f3=-Inf) end function PlantSimEngine.outputs_(::Msg3LvlScaleEchelle2Model) - (e2 = -Inf, f2 = -Inf) + (e2=-Inf, f2=-Inf) end PlantSimEngine.dep(::Msg3LvlScaleEchelle2Model) = (Msg3Lvl_echelle3=AbstractMsg3Lvl_Echelle3Model => ("E3",),) @@ -109,16 +109,16 @@ struct Msg3LvlScaleEchelle1Model <: AbstractMsg3Lvl_Echelle1Model end function PlantSimEngine.inputs_(::Msg3LvlScaleEchelle1Model) - (b = -Inf, e2 = -Inf, f2 = -Inf) + (b=-Inf, e2=-Inf, f2=-Inf) end function PlantSimEngine.outputs_(::Msg3LvlScaleEchelle1Model) - (e1 = -Inf, f1 = -Inf)#, e3 = -Inf) + (e1=-Inf, f1=-Inf)#, e3 = -Inf) end PlantSimEngine.dep(::Msg3LvlScaleEchelle1Model) = (Msg3Lvl_echelle2=AbstractMsg3Lvl_Echelle2Model => ("E2",),) function PlantSimEngine.run!(::Msg3LvlScaleEchelle1Model, models, status, meteo, constants=nothing, extra_args=nothing) - + status_E2 = extra_args.statuses["E2"][1] run!(extra_args.models["E2"].Msg3Lvl_echelle2, models, status_E2, meteo, constants, extra_args) status.e1 = status.e2 @@ -133,11 +133,11 @@ struct Msg3LvlScaleAval2Model <: AbstractMsg3Lvl_Aval2Model end function PlantSimEngine.inputs_(::Msg3LvlScaleAval2Model) - (i2 = -Inf,) + (i2=-Inf,) end - + function PlantSimEngine.outputs_(::Msg3LvlScaleAval2Model) - (g2 = -Inf,) + (g2=-Inf,) end function PlantSimEngine.run!(::Msg3LvlScaleAval2Model, models, status, meteo, constants=nothing, extra_args=nothing) @@ -150,17 +150,17 @@ struct Msg3LvlScaleAvalModel <: AbstractMsg3Lvl_AvalModel end function PlantSimEngine.inputs_(::Msg3LvlScaleAvalModel) - (e1 = -Inf, f1 = -Inf, b2 = - Inf, g2 = -Inf, e3 = -Inf) + (e1=-Inf, f1=-Inf, b2=-Inf, g2=-Inf, e3=-Inf) end - + function PlantSimEngine.outputs_(::Msg3LvlScaleAvalModel) - (g = -Inf,) + (g=-Inf,) end PlantSimEngine.dep(::Msg3LvlScaleAvalModel) = (Msg3Lvl_aval2=AbstractMsg3Lvl_Aval2Model => ("E2",),) function PlantSimEngine.run!(::Msg3LvlScaleAvalModel, models, status, meteo, constants=nothing, extra_args=nothing) - + status_E2 = extra_args.statuses["E2"][1] run!(extra_args.models["E2"].Msg3Lvl_aval2, models, status_E2, meteo, constants, extra_args) status.g = status.f1 + status.b2 + status_E2.g2 @@ -176,26 +176,26 @@ end Msg3LvlScaleAmontModel(), MultiScaleModel( model=Msg3LvlScaleAvalModel(), - mapping=[:e3 => "E3" => :e3, :b2 => "E2" => :b2, :g2 => "E2" => :g2], - ), + mapped_variables=[:e3 => "E3" => :e3, :b2 => "E2" => :b2, :g2 => "E2" => :g2], + ), MultiScaleModel( model=Msg3LvlScaleEchelle1Model(), - mapping=[:e2 => "E2" => :e2, :f2 => "E2" => :f2,], + mapped_variables=[:e2 => "E2" => :e2, :f2 => "E2" => :f2,], ), Status(a=1.0,)# y = 1.0, z = 1.0) - ), + ), "E2" => ( Msg3LvlScaleAmont2Model(), Msg3LvlScaleAval2Model(), MultiScaleModel( model=Msg3LvlScaleEchelle2Model(), - mapping=[:c => "E1" => :c, :e3 => "E3" => :e3, :f3 => "E3" => :f3,], + mapped_variables=[:c => "E1" => :c, :e3 => "E3" => :e3, :f3 => "E3" => :f3,], ), Status(a2=1.0, i2=1.0,) - ), + ), "E3" => ( MultiScaleModel( model=Msg3LvlScaleEchelle3Model(), - mapping=[:c => "E1" => :c,], + mapped_variables=[:c => "E1" => :c,], ), ), ) @@ -218,12 +218,14 @@ end Node(mtg3Lvl, MultiScaleTreeGraph.NodeMTG("/", "E2", 0, 1)) Node(mtg3Lvl, MultiScaleTreeGraph.NodeMTG("/", "E3", 0, 2)) - sim3Lvl = @test_nowarn PlantSimEngine.run!(mtg3Lvl, mapping3Lvl, meteo3Lvl, outputs=outs3Lvl, executor=SequentialEx()) + #sim3Lvl = @test_nowarn PlantSimEngine.run!(mtg3Lvl, mapping3Lvl, meteo3Lvl, tracked_outputs=outs3Lvl, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo3Lvl) + sim3Lvl = PlantSimEngine.GraphSimulation(mtg3Lvl, mapping3Lvl, nsteps=nsteps, check=false, outputs=outs3Lvl) + out = run!(sim3Lvl, meteo3Lvl) @test length(sim3Lvl.dependency_graph.roots) == 2 model_amont1 = last(collect(sim3Lvl.dependency_graph.roots)[2]) - model_ech1 = model_amont1.children[1] @test model_ech1.hard_dependency[1].children[1].parent.parent == model_ech1 @@ -236,10 +238,10 @@ end ## Hard dep at another scale, soft dep on the nested model (both at same scale) ####################################################################################################################### -PlantSimEngine.@process "hard_dep_same_scale_echelle1" verbose = false -PlantSimEngine.@process "hard_dep_same_scale_echelle1bis" verbose = false -PlantSimEngine.@process "hard_dep_same_scale_echelle3" verbose = false -PlantSimEngine.@process "hard_dep_same_scale_aval" verbose = false +PlantSimEngine.@process "hard_dep_same_scale_echelle1" verbose = false +PlantSimEngine.@process "hard_dep_same_scale_echelle1bis" verbose = false +PlantSimEngine.@process "hard_dep_same_scale_echelle3" verbose = false +PlantSimEngine.@process "hard_dep_same_scale_aval" verbose = false ################# @@ -248,11 +250,11 @@ end function PlantSimEngine.inputs_(::HardDepSameScaleEchelle3Model) #(b = -Inf, - (d = -Inf,) + (d=-Inf,) end function PlantSimEngine.outputs_(::HardDepSameScaleEchelle3Model) - (e3 = -Inf, f3 = -Inf) + (e3=-Inf, f3=-Inf) end function PlantSimEngine.run!(::HardDepSameScaleEchelle3Model, models, status, meteo, constants=nothing, extra_args=nothing) @@ -266,11 +268,11 @@ struct HardDepSameScaleEchelle1Model <: AbstractHard_Dep_Same_Scale_Echelle1Mode end function PlantSimEngine.inputs_(::HardDepSameScaleEchelle1Model) - (a = -Inf, e2 = -Inf)# e3 = -Inf, f3 = -Inf) + (a=-Inf, e2=-Inf)# e3 = -Inf, f3 = -Inf) end function PlantSimEngine.outputs_(::HardDepSameScaleEchelle1Model) - (e1 = -Inf, f1 = -Inf) + (e1=-Inf, f1=-Inf) end #PlantSimEngine.dep(::HardDepSameScaleEchelle1Model) = (hard_dep_same_scale_echelle3=AbstractHard_Dep_Same_Scale_Echelle3Model => ("E3",),) @@ -290,11 +292,11 @@ struct HardDepSameScaleEchelle1bisModel <: AbstractHard_Dep_Same_Scale_Echelle1B end function PlantSimEngine.inputs_(::HardDepSameScaleEchelle1bisModel) - (e3 = -Inf,) + (e3=-Inf,) end function PlantSimEngine.outputs_(::HardDepSameScaleEchelle1bisModel) - (e2 = -Inf, f2 = -Inf) + (e2=-Inf, f2=-Inf) end PlantSimEngine.dep(::HardDepSameScaleEchelle1bisModel) = (hard_dep_same_scale_echelle3=AbstractHard_Dep_Same_Scale_Echelle3Model => ("E3",),) @@ -314,11 +316,11 @@ struct HardDepSameScaleAvalModel <: AbstractHard_Dep_Same_Scale_AvalModel end function PlantSimEngine.inputs_(::HardDepSameScaleAvalModel) - (e3 = -Inf,) # f1 or f2 ? + (e3=-Inf,) # f1 or f2 ? end - + function PlantSimEngine.outputs_(::HardDepSameScaleAvalModel) - (g = -Inf,) + (g=-Inf,) end function PlantSimEngine.run!(::HardDepSameScaleAvalModel, models, status, meteo, constants=nothing, extra_args=nothing) @@ -334,15 +336,17 @@ end "E1" => (HardDepSameScaleEchelle1Model(), MultiScaleModel( model=HardDepSameScaleEchelle1bisModel(), - mapping=[:e3 => "E3" => :e3], + mapped_variables=[:e3 => "E3" => :e3], ), - Status(a=1.0),), + Status(a=1.0),), "E3" => ( HardDepSameScaleEchelle3Model(), HardDepSameScaleAvalModel(), Status(d=1.0,), ), ) + @test to_initialize(mapping) == Dict() + outs = Dict( "E1" => (:e1, :f1, :e2, :f2), "E3" => (:e3,) @@ -355,10 +359,13 @@ end mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "E1", 0, 0),) Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E3", 0, 1)) - sim = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, outputs=outs, executor = SequentialEx()) + #sim = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, tracked_outputs=outs, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=false, outputs=outs) + out = run!(sim, meteo) model_1 = last(collect(sim.dependency_graph.roots)[1]) - + # Downscale soft dependency aval should point to the root node 1bis, instead of the 'real parent' 3, which is an inner hard dependency to 1bis # so 1 and aval both point to 1bis @test length(model_1.children) == 2 @@ -371,7 +378,7 @@ end ## 2 different scales that make use of the *same* model ####################################################################################################################### -PlantSimEngine.@process "single_model_multiple_scales" verbose = false +PlantSimEngine.@process "single_model_multiple_scales" verbose = false struct SingleModelScale1 <: AbstractSingle_Model_Multiple_ScalesModel end @@ -383,31 +390,31 @@ struct SingleModelScale3 <: AbstractSingle_Model_Multiple_ScalesModel end function PlantSimEngine.inputs_(::SingleModelScale1) - (in = -Inf, in1 = -Inf) + (in=-Inf, in1=-Inf) end function PlantSimEngine.outputs_(::SingleModelScale1) - (out = -Inf, out1 = -Inf) + (out=-Inf, out1=-Inf) end function PlantSimEngine.inputs_(::SingleModelScale2) - (in = -Inf, in2 = -Inf) + (in=-Inf, in2=-Inf) end function PlantSimEngine.outputs_(::SingleModelScale2) - (out = -Inf, out2 = -Inf) + (out=-Inf, out2=-Inf) end function PlantSimEngine.inputs_(::SingleModelScale2bis) - (in = -Inf, in2bis = -Inf) + (in=-Inf, in2bis=-Inf) end function PlantSimEngine.outputs_(::SingleModelScale2bis) - (out = -Inf, out2bis = -Inf) + (out=-Inf, out2bis=-Inf) end function PlantSimEngine.inputs_(::SingleModelScale3) - (in = -Inf, in3 = -Inf, out2 = -Inf, out1 = -Inf) + (in=-Inf, in3=-Inf, out2=-Inf, out1=-Inf) end function PlantSimEngine.outputs_(::SingleModelScale3) - (out = -Inf, out3 = -Inf) + (out=-Inf, out3=-Inf) end PlantSimEngine.dep(::SingleModelScale1) = (single_model_multiple_scales=AbstractSingle_Model_Multiple_ScalesModel => ("E2bis", "E2"),) @@ -418,7 +425,7 @@ function PlantSimEngine.run!(::SingleModelScale1, models, status, meteo, constan status_E2b = sim_object.statuses["E2bis"][1] run!(sim_object.models["E2"].single_model_multiple_scales, models, status_E2, meteo, constants) run!(sim_object.models["E2bis"].single_model_multiple_scales, models, status_E2b, meteo, constants) - status.out = status_E2.out+ status_E2b.out + status.in + status.out = status_E2.out + status_E2b.out + status.in status.out1 = status_E2.out2 + status_E2b.out2bis + status.out1 end @@ -433,7 +440,7 @@ function PlantSimEngine.run!(::SingleModelScale2bis, models, status, meteo, cons end function PlantSimEngine.run!(::SingleModelScale3, models, status, meteo, constants=nothing, sim_object=nothing) - status.out = status.in + status.in3 + status.out2; + status.out = status.in + status.in3 + status.out2 status.out3 = status.in3 + status.out1 end @@ -443,56 +450,168 @@ end @testset "Process/model reuse at different scales" begin mapping = Dict( - "E1" => ( - SingleModelScale1(), - Status(in = 1.0, in1 = 1.0), - ), - "E2" => ( - SingleModelScale2(), - Status(in = 1.0, in2 = 1.0), - ), - "E2bis" => ( - SingleModelScale2bis(), - Status(in = 1.0, in2bis = 1.0), - ), - "E3" => ( - MultiScaleModel( - model = SingleModelScale3(), - mapping = [:out1 => "E1" => :out1, :out2 => "E2" => :out2, ], - ), - Status(in= 1.0, in3 = 1.0,), - ), + "E1" => ( + SingleModelScale1(), + Status(in=1.0, in1=1.0), + ), + "E2" => ( + SingleModelScale2(), + Status(in=1.0, in2=1.0), + ), + "E2bis" => ( + SingleModelScale2bis(), + Status(in=1.0, in2bis=1.0), + ), + "E3" => ( + MultiScaleModel( + model=SingleModelScale3(), + mapped_variables=[:out1 => "E1" => :out1, :out2 => "E2" => :out2,], + ), + Status(in=1.0, in3=1.0,), + ), + ) + + @test to_initialize(mapping) == Dict() + + outs = Dict( + "E1" => (:out, :out1), + "E2" => (:out, :out2), + "E2bis" => (:out,), # comment this line out, and remove nodes relating to E2 and E2bis to expose the issue in #103 + "E3" => (:out3,) + ) + + meteo = Weather([ + Atmosphere(T=25.0, Wind=1.0, Rh=0.6, Ri_PAR_f=200.0), + Atmosphere(T=10.0, Wind=0.5, Rh=0.6, Ri_PAR_f=200.0)]) + + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "E1", 0, 0),) + Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E3", 0, 1)) + Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E2", 0, 2)) + Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E2bis", 0, 3)) + + #sim = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, tracked_outputs=outs, executor = SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=false, outputs=outs) + out = run!(sim, meteo) + + roots = sim.dependency_graph.roots + @test length(sim.dependency_graph.roots) == 1 + + model_1 = last(collect(roots)[1]) + + @test length(model_1.children) == 1 + @test length(model_1.hard_dependency) == 2 + @test model_1.children[1].parent[1] == model_1 + @test model_1.hard_dependency[1].parent == model_1 + @test model_1.hard_dependency[2].parent == model_1 + +end + + + +########################## +## No outputs when simulating a mapping with one meteo timestep #105 +########################## + +@testset "Issue 105 : no outputs when simulating a mapping with one meteo timestep" begin + + using PlantSimEngine, PlantMeteo, DataFrames + using PlantSimEngine.Examples + mtg = import_mtg_example() + m = Dict( + "Leaf" => ( + Process1Model(1.0), + Status(var1=10.0, var2=1.0,) + ) + ) + + @test to_initialize(m) == Dict() + + vars = Dict{String,Any}("Leaf" => (:var1,)) + out = run!(mtg, m, Atmosphere(T=20.0, Wind=1.0, Rh=0.65), tracked_outputs=vars, executor=SequentialEx()) + df = convert_outputs(out, DataFrame) + @test DataFrames.nrow(df) == 2 +end + +########################## +## Multiscale : outputs not saved when dependency graph only has one depth level #111 +########################## + +# Probably very similar to #105 +@testset "Issue 111 : Multiscale : outputs not saved when dependency graph only has one depth level" begin + + using PlantSimEngine + using PlantSimEngine.Examples + using MultiScaleTreeGraph + + status2 = (var1=15.0, var2=0.3) + + meteo = Weather([ + Atmosphere(T=25.0, Wind=1.0, Rh=0.6, Ri_PAR_f=200.0), + Atmosphere(T=10.0, Wind=0.5, Rh=0.6, Ri_PAR_f=200.0)]) + + outs = Dict("Default" => (:var1,)) + mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "Default", 0, 0),) + + mapping = Dict( + "Default" => ( + Process1Model(1.0), + Status(var1=15.0, var2=0.3,), + ), ) - - outs = Dict( - "E1" => (:out, :out1), - "E2" => (:out, :out2), - "E2bis" => (:out,), # comment this line out, and remove nodes relating to E2 and E2bis to expose the issue in #103 - "E3" => (:out3,) - ) - - meteo = Weather([ - Atmosphere(T=25.0, Wind=1.0, Rh=0.6, Ri_PAR_f=200.0), - Atmosphere(T=10.0, Wind=0.5, Rh=0.6, Ri_PAR_f=200.0) - - ]) - - mtg = MultiScaleTreeGraph.Node(MultiScaleTreeGraph.NodeMTG("/", "E1", 0, 0),) - Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E3", 0, 1)) - Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E2", 0, 2)) - Node(mtg, MultiScaleTreeGraph.NodeMTG("/", "E2bis", 0, 3)) - - sim = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, outputs = outs, executor = SequentialEx()) - - roots = sim.dependency_graph.roots - @test length(sim.dependency_graph.roots) == 1 - - model_1 = last(collect(roots)[1]) - - @test length(model_1.children) == 1 - @test length(model_1.hard_dependency) == 2 - @test model_1.children[1].parent[1] == model_1 - @test model_1.hard_dependency[1].parent == model_1 - @test model_1.hard_dependency[2].parent == model_1 - - end \ No newline at end of file + @test to_initialize(mapping) == Dict() + + sim = run!(mtg, mapping, meteo; tracked_outputs=outs) + using DataFrames + df = convert_outputs(sim, DataFrame) + @test DataFrames.nrow(df) == PlantSimEngine.get_nsteps(meteo) + +end + + +############################################ +### #86 : BoundsError with a single model and several Weather timesteps +############################################ + +using PlantSimEngine +PlantSimEngine.@process "toy" verbose = false + +""" +Inputs : a, b, c +Outputs : d, e +""" + +struct ToyToyModel{T} <: AbstractToyModel + internal_constant::T +end + +function PlantSimEngine.inputs_(::ToyToyModel) + (a=-Inf, b=-Inf, c=-Inf) +end + +# note : here, d is set with = further down, but e is set with +=, ie inf + thingy, is this a bug on my end ? +function PlantSimEngine.outputs_(::ToyToyModel) + (d=-Inf, e=-Inf) +end + +function PlantSimEngine.run!(m::ToyToyModel, models, status, meteo, constants=nothing, extra_args=nothing) + status.d = m.internal_constant * status.a + status.e += m.internal_constant +end + +@testset "Issue #86 : BoundsError with a single model and several Weather timesteps" begin + meteo = Weather([ + Atmosphere(T=20.0, Wind=1.0, Rh=0.65, Ri_PAR_f=200.0), + Atmosphere(T=18.0, Wind=1.0, Rh=0.65, Ri_PAR_f=100.0), + ]) + + model = ModelList( + ToyToyModel(1), + status=(a=1, b=0, c=0), + #nsteps = length(meteo) + ) + @test to_initialize(model) == NamedTuple() + + sim = run!(model, meteo) + @test DataFrames.nrow(sim) == PlantSimEngine.get_nsteps(meteo) +end \ No newline at end of file diff --git a/test/test-dimensions.jl b/test/test-dimensions.jl index 05ac6cb5f..ab2741c52 100644 --- a/test/test-dimensions.jl +++ b/test/test-dimensions.jl @@ -13,22 +13,22 @@ @test PlantSimEngine.check_dimensions(st, atm) === nothing # TimeStepTable and Atmosphere are always authorized - @test PlantSimEngine.check_dimensions(tst1, atm) === nothing - @test PlantSimEngine.check_dimensions(tst2, atm) === nothing + # @test PlantSimEngine.check_dimensions(tst1, atm) === nothing + # @test PlantSimEngine.check_dimensions(tst2, atm) === nothing # Status and Weather are always authorized @test PlantSimEngine.check_dimensions(st, w1) === nothing @test PlantSimEngine.check_dimensions(st, w2) === nothing # TimeStepTable and Weather must be checked for equal length - @test PlantSimEngine.check_dimensions(tst1, w1) === nothing - @test PlantSimEngine.check_dimensions(tst2, w2) === nothing + # @test PlantSimEngine.check_dimensions(tst1, w1) === nothing + # @test PlantSimEngine.check_dimensions(tst2, w2) === nothing # This still works because one time step is recycled: - @test PlantSimEngine.check_dimensions(tst1, w2) === nothing + # @test PlantSimEngine.check_dimensions(tst1, w2) === nothing - @test_throws DimensionMismatch PlantSimEngine.check_dimensions(tst2, w1) - @test_throws DimensionMismatch PlantSimEngine.check_dimensions(tst3, w2) + # @test_throws DimensionMismatch PlantSimEngine.check_dimensions(tst2, w1) + # @test_throws DimensionMismatch PlantSimEngine.check_dimensions(tst3, w2) # ModelList and Weather must be checked for equal length m1 = ModelList( diff --git a/test/test-fitting.jl b/test/test-fitting.jl index d5898b493..dfb685c3b 100644 --- a/test/test-fitting.jl +++ b/test/test-fitting.jl @@ -4,11 +4,11 @@ k = 0.6 meteo = Atmosphere(T=20.0, Wind=1.0, P=101.3, Rh=0.65, Ri_PAR_f=300.0) m = ModelList(Beer(k), status=(LAI=2.0,)) - run!(m, meteo) + outs = run!(m, meteo) - df = DataFrame(aPPFD=m[:aPPFD][1], LAI=m.status.LAI[1], Ri_PAR_f=meteo.Ri_PAR_f[1]) + df = DataFrame(aPPFD=outs[:aPPFD][1], LAI=m.status.LAI[1], Ri_PAR_f=meteo.Ri_PAR_f[1]) - k_fit = fit(Beer, df).k + k_fit = fit(PlantSimEngine.Examples.Beer, df).k @test k_fit == k end; diff --git a/test/test-mapping.jl b/test/test-mapping.jl index b5921d9fe..a96c6aa80 100755 --- a/test/test-mapping.jl +++ b/test/test-mapping.jl @@ -2,7 +2,7 @@ mapping = Dict( "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -12,7 +12,7 @@ mapping = Dict( ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( @@ -23,7 +23,7 @@ mapping = Dict( "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -111,7 +111,7 @@ function modellist_to_mapping_manual(modellist_original::ModelList, modellist_st PlantSimEngine.HelperNextTimestepModel(), MultiScaleModel( model=PlantSimEngine.HelperCurrentTimestepModel(), - mapping=[PreviousTimeStep(:next_timestep),], + mapped_variables=[PreviousTimeStep(:next_timestep),], ), Status(current_timestep=1,next_timestep=1) ), @@ -175,7 +175,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyTestDegreeDaysCumulModel}) = Pl status=st, ) - run!(models, + modellist_outputs = run!(models, meteo_day ; check=true, @@ -193,16 +193,16 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyTestDegreeDaysCumulModel}) = Pl executor=SequentialEx() ) - @test compare_outputs_modellist_mapping(models, graphsim) + @test compare_outputs_modellist_mapping(modellist_outputs, graphsim) # fully automated model generation st2 = (TT_cu=Vector(cumsum(meteo_day.TT)),) - # TODO outputs name conflict if this is just named outputs - # TODO when outputs filtering is implemented, can test it with this function mtg, mapping, outputs_mapping = PlantSimEngine.modellist_to_mapping(models, st2; nsteps=nsteps, outputs=nothing) - graphsim2 = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=outputs_mapping) + @test to_initialize(mapping) == Dict() + + graphsim2 = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=outputs_mapping) sim2 = run!(graphsim2, meteo_day, @@ -211,7 +211,7 @@ PlantSimEngine.ObjectDependencyTrait(::Type{<:ToyTestDegreeDaysCumulModel}) = Pl check=true, executor=SequentialEx() ) - @test compare_outputs_modellist_mapping(models, graphsim2) + @test compare_outputs_modellist_mapping(modellist_outputs, graphsim2) @test compare_outputs_graphsim(graphsim, graphsim2) end @@ -230,7 +230,7 @@ end "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -240,7 +240,7 @@ end ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( @@ -251,7 +251,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -268,6 +268,8 @@ mtg = import_mtg_example(); mapping_without_vectors = PlantSimEngine.replace_mapping_status_vectors_with_generated_models(mapping_with_vector, "Soil", nsteps) +@test to_initialize(mapping_without_vectors) == Dict() + graph_sim_multiscale = @test_nowarn PlantSimEngine.GraphSimulation(mtg, mapping_without_vectors, nsteps=nsteps, check=true, outputs=out_multiscale) sim_multiscale = run!(graph_sim_multiscale, @@ -277,4 +279,60 @@ mapping_without_vectors = PlantSimEngine.replace_mapping_status_vectors_with_gen check=true, executor=SequentialEx() ) + + #replace a value with a constant vector and ensure no changes happen in the simulation + carbon_biomass_vec = Vector{Float64}(undef, nsteps) + for i in nsteps + carbon_biomass_vec[i] = 2.0 + end + mapping_with_two_vectors = Dict("Plant" => ( + MultiScaleModel( + model=ToyCAllocationModel(), + mapped_variables=[ + # inputs + :carbon_assimilation => ["Leaf"], + :carbon_demand => ["Leaf", "Internode"], + # outputs + :carbon_allocation => ["Leaf", "Internode"] + ], + ), + MultiScaleModel( + model=ToyPlantRmModel(), + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + ), + ), + "Internode" => ( + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), + Status(TT=TT_v, carbon_biomass=1.0) + ), + "Leaf" => ( + MultiScaleModel( + model=ToyAssimModel(), + mapped_variables=[:soil_water_content => "Soil",], + ), + ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), + ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), + Status(aPPFD=1300.0, carbon_biomass=carbon_biomass_vec, TT=10.0), # Replaced with vector here + ), + "Soil" => ( + ToySoilWaterModel(), + ), + ) + + mtg = import_mtg_example() + mapping_without_vectors_2 = PlantSimEngine.replace_mapping_status_vectors_with_generated_models(mapping_with_two_vectors, "Soil", nsteps) + graph_sim_multiscale_2 = @test_nowarn PlantSimEngine.GraphSimulation(mtg, mapping_without_vectors_2, nsteps=nsteps, check=true, outputs=out_multiscale) + + @test to_initialize(mapping_without_vectors_2) == Dict() + + sim_multiscale_2 = run!(graph_sim_multiscale_2, + meteo_day, + PlantMeteo.Constants(), + nothing; + check=true, + executor=SequentialEx() + ) + + @test compare_outputs_graphsim(graph_sim_multiscale, graph_sim_multiscale_2) end \ No newline at end of file diff --git a/test/test-mtg-dynamic.jl b/test/test-mtg-dynamic.jl index bd044b89d..b767c391e 100644 --- a/test/test-mtg-dynamic.jl +++ b/test/test-mtg-dynamic.jl @@ -15,14 +15,14 @@ mapping = Dict( "Plant" => ( MultiScaleModel( model=ToyLAIModel(), - mapping=[ + mapped_variables=[ :TT_cu => "Scene", ], ), Beer(0.6), MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] @@ -30,17 +30,17 @@ mapping = Dict( ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), MultiScaleModel( model=ToyInternodeEmergence(TT_emergence=20.0), - mapping=[:TT_cu => "Scene"], + mapped_variables=[:TT_cu => "Scene"], ), ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), Status(carbon_biomass=1.0) @@ -48,11 +48,11 @@ mapping = Dict( "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil", :aPPFD => "Plant"], + mapped_variables=[:soil_water_content => "Soil", :aPPFD => "Plant"], ), MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), Status(carbon_biomass=1.0) @@ -69,17 +69,20 @@ out_vars = Dict( "Soil" => (:soil_water_content,), ) -out = run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()) +nsteps = PlantSimEngine.get_nsteps(meteo) +sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) +out = run!(sim,meteo) +#out = run!(mtg, mapping, meteo, tracked_outputs=out_vars, executor=SequentialEx()) @testset "MTG with dynamic growth" begin - st = out.statuses + st = sim.statuses @test length(mtg) == 9 @test length(st["Scene"]) == length(st["Soil"]) == length(st["Plant"]) == 1 @test length(st["Internode"]) == length(st["Leaf"]) == 3 @test st["Internode"][1].TT_cu_emergence == 0.0 @test st["Internode"][end].TT_cu_emergence == 25.0 - out_df = outputs(out, DataFrame) + out_df = convert_outputs(out, DataFrame) @test unique(out_df[:, :organ]) |> sort == ["Internode", "Leaf", "Plant", "Soil"] @test filter(row -> row.organ == "Internode", out_df)[:, :TT_cu_emergence] == [0.0, 0.0, 0.0, 0.0, 25.0] @test filter(row -> row.organ == "Leaf", out_df)[:, :carbon_demand] == [0.5, 0.5, 0.75, 0.75, 0.75] diff --git a/test/test-mtg-multiscale-cyclic-dep.jl b/test/test-mtg-multiscale-cyclic-dep.jl index 64c4ecaa7..134941e86 100644 --- a/test/test-mtg-multiscale-cyclic-dep.jl +++ b/test/test-mtg-multiscale-cyclic-dep.jl @@ -19,14 +19,14 @@ out_vars = Dict( "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] ], ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6), ), @@ -61,14 +61,14 @@ end "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] ], ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), Status(total_surface=0.001, aPPFD=1300.0, soil_water_content=0.6, carbon_assimilation=5.0), ), @@ -76,7 +76,7 @@ end ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), MultiScaleModel( model=ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) ), Status(TT=10.0, carbon_biomass=1.0), ), @@ -84,7 +84,7 @@ end ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), MultiScaleModel( model=ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (second break) + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (second break) ), ToyCBiomassModel(1.2), Status(TT=10.0), @@ -104,8 +104,12 @@ end @test length(cycle_vec) == 7 @test to_initialize(mapping_nocyclic) == Dict() - out = @test_nowarn run!(mtg, mapping_nocyclic, meteo, outputs=out_vars, executor=SequentialEx()) - st = status(out) + + #out = @test_nowarn run!(mtg, mapping_nocyclic, meteo, tracked_outputs=out_vars, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) + out = @test_nowarn run!(sim,meteo) + st = status(sim) st["Leaf"][1].carbon_biomass = 2.0 @test st["Leaf"][2].carbon_biomass != 2.0 @@ -117,7 +121,7 @@ end ToyDegreeDaysCumulModel(), MultiScaleModel( model=ToyLAIfromLeafAreaModel(1.0), - mapping=[ + mapped_variables=[ :plant_surfaces => ["Plant" => :surface], ], ), @@ -126,7 +130,7 @@ end "Plant" => ( MultiScaleModel( model=ToyPlantLeafSurfaceModel(), - mapping=[PreviousTimeStep(:leaf_surfaces) => ["Leaf" => :surface],], + mapped_variables=[PreviousTimeStep(:leaf_surfaces) => ["Leaf" => :surface],], #! We use PreviousTimeStep to break the cyclic dependency between the LAI and the leaf surface # that is computed as one of the latest sub-models. Now the LAI used for light interception # will be the one from the previous time-step, and at the end of the time-step we will update @@ -134,41 +138,41 @@ end ), MultiScaleModel( model=ToyLightPartitioningModel(), - mapping=[ + mapped_variables=[ :aPPFD_larger_scale => "Scene" => :aPPFD, :total_surface => "Scene" ], ), MultiScaleModel( model=ToyAssimModel(), - mapping=[ + mapped_variables=[ :soil_water_content => "Soil", ], ), MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ :carbon_demand => ["Leaf", "Internode"], :carbon_allocation => ["Leaf", "Internode"] ], ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), MultiScaleModel( model=ToyInternodeEmergence(TT_emergence=20.0), - mapping=[:TT_cu => "Scene"], + mapped_variables=[:TT_cu => "Scene"], ), MultiScaleModel( model=ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) ), ToyCBiomassModel(1.1), Status(carbon_biomass=0.0) @@ -176,11 +180,11 @@ end "Leaf" => ( MultiScaleModel( model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], + mapped_variables=[:TT => "Scene",], ), MultiScaleModel( model=ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - mapping=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) + mapped_variables=[PreviousTimeStep(:carbon_biomass),], #! this is where we break the cyclic dependency (first break) ), ToyCBiomassModel(1.2), ToyLeafSurfaceModel(0.1), @@ -212,10 +216,15 @@ end d = @test_nowarn dep(mapping) @test to_initialize(mapping) == Dict() - out = @test_nowarn run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()) + #out = @test_nowarn run!(mtg, mapping, meteo, tracked_outputs=out_vars, executor=SequentialEx()) + + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) + out = run!(sim,meteo) + # To update the reference: ref_path = joinpath(pkgdir(PlantSimEngine), "test/references/ref_output_simulation.csv") - # CSV.write(ref_path, sort(outputs(out, DataFrame, no_value=missing), [:timestep, :node]), transform=(col, val) -> something(val, missing)) + # CSV.write(ref_path, sort(convert_outputs(out, DataFrame, no_value=missing), [:timestep, :node]), transform=(col, val) -> something(val, missing)) ref_df = CSV.read(ref_path, DataFrame) - @test isequal(sort(outputs(out, DataFrame, no_value=missing), [:timestep, :node]), ref_df) + @test isequal(sort(convert_outputs(out, DataFrame, no_value=missing), [:timestep, :node]), ref_df) end \ No newline at end of file diff --git a/test/test-mtg-multiscale.jl b/test/test-mtg-multiscale.jl index cd3dd7914..d1113f46b 100644 --- a/test/test-mtg-multiscale.jl +++ b/test/test-mtg-multiscale.jl @@ -35,7 +35,7 @@ mapping_1 = Dict( "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -45,7 +45,7 @@ mapping_1 = Dict( ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( @@ -56,7 +56,7 @@ mapping_1 = Dict( "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -224,7 +224,7 @@ end "Plant" => MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -236,7 +236,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -265,7 +265,7 @@ end "Plant" => MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -277,7 +277,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -306,7 +306,7 @@ end "Plant" => MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -318,7 +318,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -337,7 +337,7 @@ end "Plant" => MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -349,7 +349,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil" => :var3,], + mapped_variables=[:soil_water_content => "Soil" => :var3,], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -373,23 +373,32 @@ end end # Testing with a simple mapping (just the soil model, no multiscale mapping): @testset "run! on MTG: simple mapping" begin - out = @test_nowarn run!(mtg, Dict("Soil" => (ToySoilWaterModel(),)), meteo) - @test out.statuses["Soil"][1].node == soil - @test out.models == Dict("Soil" => (soil_water=ToySoilWaterModel(out.models["Soil"].soil_water.values),)) - @test out.models["Soil"].soil_water.values == [0.5] - @test length(out.dependency_graph.roots) == 1 - @test collect(keys(out.dependency_graph.roots))[1] == Pair("Soil", :soil_water) - @test out.graph == mtg + #out = @test_nowarn run!(mtg, Dict("Soil" => (ToySoilWaterModel(),)), meteo) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, Dict("Soil" => (ToySoilWaterModel(),)), nsteps=nsteps, check=true, outputs=nothing) + out = run!(sim,meteo) + + @test sim.statuses["Soil"][1].node == soil + @test sim.models == Dict("Soil" => (soil_water=ToySoilWaterModel(sim.models["Soil"].soil_water.values),)) + @test sim.models["Soil"].soil_water.values == [0.5] + @test length(sim.dependency_graph.roots) == 1 + @test collect(keys(sim.dependency_graph.roots))[1] == Pair("Soil", :soil_water) + @test sim.graph == mtg leaf_mapping = Dict("Leaf" => (ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), Status(TT=10.0))) - out = run!(mtg, leaf_mapping, meteo) - @test collect(keys(out.statuses)) == ["Leaf"] - @test length(out.statuses["Leaf"]) == 2 - @test out.statuses["Leaf"][1].TT == 10.0 # As initialized in the mapping - @test out.statuses["Leaf"][1].carbon_demand == 0.5 - - @test out.statuses["Leaf"][1].node == leaf1 - @test out.statuses["Leaf"][2].node == leaf2 + + #out = run!(mtg, leaf_mapping, meteo) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, leaf_mapping, nsteps=nsteps, check=true, outputs=nothing) + out = run!(sim,meteo) + + @test collect(keys(sim.statuses)) == ["Leaf"] + @test length(sim.statuses["Leaf"]) == 2 + @test sim.statuses["Leaf"][1].TT == 10.0 # As initialized in the mapping + @test sim.statuses["Leaf"][1].carbon_demand == 0.5 + + @test sim.statuses["Leaf"][1].node == leaf1 + @test sim.statuses["Leaf"][2].node == leaf2 end # A mapping with all different types of mapping (single, multi-scale, model as is, or tuple of): @@ -398,7 +407,7 @@ end "Plant" => MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -410,7 +419,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -422,91 +431,101 @@ end ) # The mapping above should throw an error because TT is not initialized for the Internode: if VERSION < v"1.8" # We test differently depending on the julia version because the format of the error message changed - @test_throws ErrorException run!(mtg, mapping_all, meteo) + + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping_all, nsteps=nsteps, check=true, outputs=nothing) + @test_throws ErrorException out = run!(sim,meteo) else @test_throws "Variable `Rm` is not computed by any model, not initialised by the user in the status, and not found in the MTG at scale Plant (checked for MTG node 3)." run!(mtg, mapping_all, meteo) end # It should work if we don't check the mapping though: - out = @test_nowarn run!(mtg, mapping_all, meteo, check=false) + #out = @test_nowarn run!(mtg, mapping_all, meteo, check=false) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping_all, nsteps=nsteps, check=false, outputs=nothing) + out = run!(sim,meteo) # Note that the outputs are garbage because the TT is not initialized. - @test out.models == Dict{String,NamedTuple}( - "Soil" => (soil_water=ToySoilWaterModel(out.models["Soil"].soil_water.values),), + @test sim.models == Dict{String,NamedTuple}( + "Soil" => (soil_water=ToySoilWaterModel(sim.models["Soil"].soil_water.values),), "Internode" => (carbon_demand=ToyCDemandModel{Float64}(10.0, 200.0),), "Plant" => (carbon_allocation=ToyCAllocationModel(),), "Leaf" => (carbon_assimilation=ToyAssimModel{Float64}(0.2), carbon_demand=ToyCDemandModel{Float64}(10.0, 200.0)) ) - @test out.models["Soil"].soil_water.values == [0.5] - @test length(out.dependency_graph.roots) == 3 # 3 because the plant is not a root (its model has dependencies) - @test out.statuses["Internode"][1].TT === -Inf - @test out.statuses["Internode"][1].carbon_demand === -Inf + @test sim.models["Soil"].soil_water.values == [0.5] + @test length(sim.dependency_graph.roots) == 3 # 3 because the plant is not a root (its model has dependencies) + @test sim.statuses["Internode"][1].TT === -Inf + @test sim.statuses["Internode"][1].carbon_demand === -Inf - st_leaf1 = out.statuses["Leaf"][1] + st_leaf1 = sim.statuses["Leaf"][1] @test st_leaf1.TT == 10.0 @test st_leaf1.carbon_demand == 0.5 # This one depends on the soil, which is random, so we test using the computation directly: - @test st_leaf1.carbon_assimilation == st_leaf1.aPPFD * out.models["Leaf"].carbon_assimilation.LUE * st_leaf1.soil_water_content + @test st_leaf1.carbon_assimilation == st_leaf1.aPPFD * sim.models["Leaf"].carbon_assimilation.LUE * st_leaf1.soil_water_content @test st_leaf1.carbon_allocation == 0.0 end @testset "run! on MTG with complete mapping (with init)" begin - out = @test_nowarn run!(mtg_init, mapping_1, meteo, executor=SequentialEx()) - - @test typeof(out.statuses) == Dict{String,Vector{Status}} - @test length(out.statuses["Plant"]) == 1 - @test length(out.statuses["Leaf"]) == 2 - @test length(out.statuses["Internode"]) == 2 - @test length(out.statuses["Soil"]) == 1 - @test out.statuses["Soil"][1].node == get_node(mtg_init, 2) - @test out.statuses["Soil"][1].soil_water_content !== -Inf + + #out = @test_nowarn run!(mtg_init, mapping_1, meteo, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg_init, mapping_1, nsteps=nsteps, check=false, outputs=nothing) + out = run!(sim,meteo) + + @test typeof(sim.statuses) == Dict{String,Vector{Status}} + @test length(sim.statuses["Plant"]) == 1 + @test length(sim.statuses["Leaf"]) == 2 + @test length(sim.statuses["Internode"]) == 2 + @test length(sim.statuses["Soil"]) == 1 + @test sim.statuses["Soil"][1].node == get_node(mtg_init, 2) + @test sim.statuses["Soil"][1].soil_water_content !== -Inf # Testing that we get the link between the node and its status: - @test out.statuses["Soil"][1] == get_node(mtg_init, 2).plantsimengine_status + @test sim.statuses["Soil"][1] == get_node(mtg_init, 2).plantsimengine_status # Testing if the value in the status of the leaves is the same as the one in the status of the soil: - @test out.statuses["Soil"][1].soil_water_content === out.statuses["Leaf"][1].soil_water_content - @test out.statuses["Soil"][1].soil_water_content === out.statuses["Leaf"][2].soil_water_content + @test sim.statuses["Soil"][1].soil_water_content === sim.statuses["Leaf"][1].soil_water_content + @test sim.statuses["Soil"][1].soil_water_content === sim.statuses["Leaf"][2].soil_water_content - leaf1_status = out.statuses["Leaf"][1] + leaf1_status = sim.statuses["Leaf"][1] # This is the model that computes the assimilation (testing manually that we get the right result here): - @test leaf1_status.carbon_assimilation == leaf1_status.aPPFD * out.models["Leaf"].carbon_assimilation.LUE * leaf1_status.soil_water_content + @test leaf1_status.carbon_assimilation == leaf1_status.aPPFD * sim.models["Leaf"].carbon_assimilation.LUE * leaf1_status.soil_water_content - @test out.statuses["Plant"][1].carbon_demand[[1, 3]] == [i.carbon_demand for i in out.statuses["Internode"]] - @test out.statuses["Plant"][1].carbon_demand[[2, 4]] == [i.carbon_demand for i in out.statuses["Leaf"]] + @test sim.statuses["Plant"][1].carbon_demand[[1, 3]] == [i.carbon_demand for i in sim.statuses["Internode"]] + @test sim.statuses["Plant"][1].carbon_demand[[2, 4]] == [i.carbon_demand for i in sim.statuses["Leaf"]] # Testing the reference directly: - ref_values_cdemand = getfield(out.statuses["Plant"][1].carbon_demand, :v) + ref_values_cdemand = getfield(sim.statuses["Plant"][1].carbon_demand, :v) for (j, i) in enumerate([1, 3]) - @test ref_values_cdemand[i] === PlantSimEngine.refvalue(out.statuses["Internode"][j], :carbon_demand) + @test ref_values_cdemand[i] === PlantSimEngine.refvalue(sim.statuses["Internode"][j], :carbon_demand) end for (j, i) in enumerate([2, 4]) - @test ref_values_cdemand[i] === PlantSimEngine.refvalue(out.statuses["Leaf"][j], :carbon_demand) + @test ref_values_cdemand[i] === PlantSimEngine.refvalue(sim.statuses["Leaf"][j], :carbon_demand) end # Testing that carbon allocation in Leaf and Internode was added as a variable from the model at the Plant scale: - @test hasproperty(out.statuses["Internode"][1], :carbon_allocation) - @test hasproperty(out.statuses["Leaf"][1], :carbon_allocation) + @test hasproperty(sim.statuses["Internode"][1], :carbon_allocation) + @test hasproperty(sim.statuses["Leaf"][1], :carbon_allocation) - @test out.statuses["Internode"][1].carbon_allocation == 0.5 - @test out.statuses["Leaf"][1].carbon_allocation == 0.5 + @test sim.statuses["Internode"][1].carbon_allocation == 0.5 + @test sim.statuses["Leaf"][1].carbon_allocation == 0.5 # Testing that we get the link between the node and its status: - @test out.statuses["Leaf"][2] == get_node(mtg_init, 7).plantsimengine_status + @test sim.statuses["Leaf"][2] == get_node(mtg_init, 7).plantsimengine_status # Testing the reference directly: - ref_values_callocation = getfield(out.statuses["Plant"][1].carbon_allocation, :v) + ref_values_callocation = getfield(sim.statuses["Plant"][1].carbon_allocation, :v) for (j, i) in enumerate([1, 3]) - @test ref_values_callocation[i] === PlantSimEngine.refvalue(out.statuses["Internode"][j], :carbon_allocation) + @test ref_values_callocation[i] === PlantSimEngine.refvalue(sim.statuses["Internode"][j], :carbon_allocation) end for (j, i) in enumerate([2, 4]) - @test ref_values_callocation[i] === PlantSimEngine.refvalue(out.statuses["Leaf"][j], :carbon_allocation) + @test ref_values_callocation[i] === PlantSimEngine.refvalue(sim.statuses["Leaf"][j], :carbon_allocation) end end @@ -537,12 +556,15 @@ end ) ) - out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) + #out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=false, outputs=nothing) + out = run!(sim,meteo) - @test out.statuses["Leaf"][1].var1 === var1 - @test out.statuses["Leaf"][1].var2 === 1.0 - @test out.statuses["Leaf"][1].var3 === 2.0 - @test out.statuses["Leaf"][1].var6 === 40.4 + @test sim.statuses["Leaf"][1].var1 === var1 + @test sim.statuses["Leaf"][1].var2 === 1.0 + @test sim.statuses["Leaf"][1].var3 === 2.0 + @test sim.statuses["Leaf"][1].var6 === 40.4 end @testset "MTG with complex mapping" begin @@ -551,7 +573,7 @@ end "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -561,7 +583,7 @@ end ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( @@ -572,7 +594,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -590,14 +612,17 @@ end ), ) - out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) - - @test length(out.dependency_graph.roots) == 6 - @test out.statuses["Leaf"][1].var1 === 1.01 - @test out.statuses["Leaf"][1].var2 === 1.03 - @test out.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 - @test out.statuses["Leaf"][1].var5 == 32.4806 - @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 + #out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=false, outputs=nothing) + out = run!(sim,meteo) + + @test length(sim.dependency_graph.roots) == 6 + @test sim.statuses["Leaf"][1].var1 === 1.01 + @test sim.statuses["Leaf"][1].var2 === 1.03 + @test sim.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 + @test sim.statuses["Leaf"][1].var5 == 32.4806 + @test sim.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 end @testset "MTG with dynamic output variables" begin @@ -606,7 +631,7 @@ end "Plant" => ( MultiScaleModel( model=ToyCAllocationModel(), - mapping=[ + mapped_variables=[ # inputs :carbon_assimilation => ["Leaf"], :carbon_demand => ["Leaf", "Internode"], @@ -616,7 +641,7 @@ end ), MultiScaleModel( model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], + mapped_variables=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], ), ), "Internode" => ( @@ -627,7 +652,7 @@ end "Leaf" => ( MultiScaleModel( model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil",], + mapped_variables=[:soil_water_content => "Soil",], # Notice we provide "Soil", not ["Soil"], so a single value is expected here ), ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), @@ -651,24 +676,28 @@ end "Plant" => (:carbon_allocation,), "Soil" => (:soil_water_content,), ) - out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, outputs=out_vars, executor=SequentialEx()) - - @test length(out.dependency_graph.roots) == 6 - @test out.statuses["Leaf"][1].var1 === 1.01 - @test out.statuses["Leaf"][1].var2 === 1.03 - @test out.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 - @test out.statuses["Leaf"][1].var5 == 32.4806 - @test out.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 - - @test out.outputs["Leaf"][:carbon_demand] == [[0.5, 0.5], [0.5, 0.5]] - @test out.outputs["Leaf"][:soil_water_content][1] == fill(out.outputs["Soil"][:soil_water_content][1][1], 2) - @test out.outputs["Leaf"][:soil_water_content][2] == fill(out.outputs["Soil"][:soil_water_content][2][1], 2) - - @test out.outputs["Leaf"][:carbon_allocation] == out.outputs["Internode"][:carbon_allocation] - @test out.outputs["Plant"][:carbon_allocation][1][1][1] === out.outputs["Internode"][:carbon_allocation][1][1] + + #out = @test_nowarn PlantSimEngine.run!(mtg, mapping, meteo, tracked_outputs=out_vars, executor=SequentialEx()) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true, outputs=out_vars) + out = run!(sim,meteo) + + @test length(sim.dependency_graph.roots) == 6 + @test sim.statuses["Leaf"][1].var1 === 1.01 + @test sim.statuses["Leaf"][1].var2 === 1.03 + @test sim.statuses["Leaf"][1].var4 ≈ 8.1612000000000013 atol = 1e-6 + @test sim.statuses["Leaf"][1].var5 == 32.4806 + @test sim.statuses["Leaf"][1].var8 ≈ 1321.0700490800002 atol = 1e-6 + + @test sim.outputs["Leaf"][:carbon_demand] == [[0.5, 0.5], [0.5, 0.5]] + @test sim.outputs["Leaf"][:soil_water_content][1] == fill(sim.outputs["Soil"][:soil_water_content][1][1], 2) + @test sim.outputs["Leaf"][:soil_water_content][2] == fill(sim.outputs["Soil"][:soil_water_content][2][1], 2) + + @test sim.outputs["Leaf"][:carbon_allocation] == sim.outputs["Internode"][:carbon_allocation] + @test sim.outputs["Plant"][:carbon_allocation][1][1][1] === sim.outputs["Internode"][:carbon_allocation][1][1] # Testing the outputs if transformed into a DataFrame: - outs = outputs(out, DataFrame) + outs = convert_outputs(out, DataFrame) @test isa(outs, DataFrame) @test size(outs) == (12, 7) diff --git a/test/test-performance.jl b/test/test-performance.jl index 781a27ac7..e09541a2f 100644 --- a/test/test-performance.jl +++ b/test/test-performance.jl @@ -17,158 +17,97 @@ end PlantSimEngine.TimeStepDependencyTrait(::Type{<:ToySleepModel}) = PlantSimEngine.IsTimeStepIndependent() -@testset begin "Check number of threads" - nthr = Threads.nthreads() - @test nthr == 4 - - meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) +meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) nrows = nrow(meteo_day) vc = [0 for i in 1:nrows] - models1 = ModelList(process1=ToySleepModel(), status=(a=vc,)) - models2 = ModelList(process1=ToySleepModel(), status=(a=vc,)) +models1 = ModelList(process1=ToySleepModel(), status=(a=vc,)) +models2 = ModelList(process1=ToySleepModel(), status=(a=vc,)) +@testset begin "Check number of threads" + nthr = Threads.nthreads() + @test nthr > 1 + t_seq = @benchmark run!(models1, meteo_day; executor = SequentialEx()) #t_seq = run!(models1, meteo_day; executor = SequentialEx()) - min_time_seq = minimum(t_seq).time + med_time_seq = median(t_seq).time #time is in nanoseconds - @test min_time_seq > nrows * 1000000 + @test med_time_seq > nrows * 1000000 t_mt = @benchmark run!(models2, meteo_day; executor = ThreadedEx()) #t_mt = run!(models2, meteo_day; executor = ThreadedEx()) - min_time_mt = minimum(t_mt).time - - @test min_time_mt > nrows * 1000000 / nthr - - # expecting mt to have some overhead - @test nthr * min_time_mt > min_time_seq - - # todo DataFrame equals - @test status(models1) == status(models2) -end - - -############################################# -### Simulation with many organs in the MTG (but only a few different types of organs) - - -PlantSimEngine.@process "organ_crazy_emergence" verbose = false - -""" - ToyInternodeCrazyEmergence(;init_TT=0.0, TT_emergence = 300) + med_time_mt = median(t_mt).time -Computes the organ emergence based on cumulated thermal time since last event. -""" -struct ToyInternodeCrazyEmergence <: AbstractOrgan_Crazy_EmergenceModel - TT_emergence::Float64 -end - -ToyInternodeCrazyEmergence(; TT_emergence=300.0) = ToyInternodeCrazyEmergence(TT_emergence) - -PlantSimEngine.inputs_(m::ToyInternodeCrazyEmergence) = (TT_cu=-Inf,) -PlantSimEngine.outputs_(m::ToyInternodeCrazyEmergence) = (TT_cu_emergence=0.0,) + @test med_time_mt > nrows * 1000000 / nthr -function PlantSimEngine.run!(m::ToyInternodeCrazyEmergence, models, status, meteo, constants=nothing, sim_object=nothing) + # Threads sleep/wakeup scheduling overhead causing inconsistencies ? + # In any case, sometimes MT beats ST on CI runners, and the mac runner seems to return puzzling false positives + # Deactivating it for now + # TODO there is a thread discussing unreliability of the sleep() function, need to check it - #root = get_root(status.node) - - #if nleaves(root) > 10000 - # return nothing + #if !Sys.isapple() + # @test abs(nthr * med_time_mt - med_time_seq) < 0.2 * med_time_seq #end - if length(MultiScaleTreeGraph.children(status.node)) == 1 && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence - - status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) - add_organ!(status_new_internode.node, sim_object, "+", "Leaf", 2, index=1) - status_new_internode.TT_cu_emergence = status.TT_cu - elseif (length(MultiScaleTreeGraph.children(status.node)) >= 2 && length(MultiScaleTreeGraph.children(status.node)) < 7) && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence - status_new_internode = add_organ!(status.node, sim_object, "<", "Internode", 2, index=1) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=4) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=5) - status_new_internode.TT_cu_emergence = status.TT_cu - elseif (length(MultiScaleTreeGraph.children(status.node)) >= 7 && length(MultiScaleTreeGraph.children(status.node)) < 30) && status.TT_cu - status.TT_cu_emergence >= m.TT_emergence - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=6) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=7) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=8) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=9) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=10) - add_organ!(status.node, sim_object, "+", "Leaf", 2, index=11) - - end - - return nothing + # unsure how to recover outputs in benchmarked expressions to compare them, rerun the functions as a workaround for now + @test run!(models1, meteo_day; executor = SequentialEx()) == run!(models2, meteo_day; executor = ThreadedEx()) end +# TODO make sure a mt test with nthreads == 1 also is tested and is correct +@testset "Single and multi-threaded output consistency" begin + nthr = Threads.nthreads() + @test nthr == 4 -# Wrapped this into a function so that it doesn't plague the benchmark with variables on a global scope -#@check_allocs -function do_benchmark_on_heavier_mtg() - mtg = import_mtg_example(); - - # Example meteo, 365 timesteps : + using Dates meteo_day = read_weather(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), duration=Day) - - #similar to the mtg growth test but with a much lower emergence threshold - mapping = Dict( - "Scene" => ToyDegreeDaysCumulModel(), - "Plant" => ( - MultiScaleModel( - model=ToyLAIModel(), - mapping=[ - :TT_cu => "Scene", - ], - ), - Beer(0.6), - MultiScaleModel( - model=ToyCAllocationModel(), - mapping=[ - :carbon_assimilation => ["Leaf"], - :carbon_demand => ["Leaf", "Internode"], - :carbon_allocation => ["Leaf", "Internode"] - ], - ), - MultiScaleModel( - model=ToyPlantRmModel(), - mapping=[:Rm_organs => ["Leaf" => :Rm, "Internode" => :Rm],], - ), - ), - "Internode" => ( - MultiScaleModel( - model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], - ), - MultiScaleModel( - model=ToyInternodeCrazyEmergence(TT_emergence=1.0), - mapping=[:TT_cu => "Scene"], - ), - ToyMaintenanceRespirationModel(1.5, 0.06, 25.0, 0.6, 0.004), - Status(carbon_biomass=1.0) - ), - "Leaf" => ( - MultiScaleModel( - model=ToyAssimModel(), - mapping=[:soil_water_content => "Soil", :aPPFD => "Plant"], - ), - MultiScaleModel( - model=ToyCDemandModel(optimal_biomass=10.0, development_duration=200.0), - mapping=[:TT => "Scene",], - ), - ToyMaintenanceRespirationModel(2.1, 0.06, 25.0, 1.0, 0.025), - Status(carbon_biomass=1.0) - ), - "Soil" => ( - ToySoilWaterModel(), - ), + + models = ModelList( + ToyLAIModel(), + Beer(0.5), + status=(TT_cu=cumsum(meteo_day.TT),), ) + + tracked_outputs = (:LAI,) - out_vars = Dict( - "Leaf" => (:carbon_assimilation, :carbon_demand, :soil_water_content, :carbon_allocation), - "Internode" => (:carbon_allocation, :TT_cu_emergence), - "Plant" => (:carbon_allocation,), - "Soil" => (:soil_water_content,), - ) + out_seq, out_mt = run_single_and_multi_thread_modellist(models, tracked_outputs, meteo_day) + @test compare_outputs_modellists(out_seq, out_mt) + + modellists, status_tuples, outs_vectors = get_modellist_bank() + meteos_all = get_simple_meteo_bank() - out = run!(mtg, mapping, meteo_day, outputs=out_vars, executor=SequentialEx()); -end + # First meteo only has one timestep + meteos = meteos_all[2:length(meteos_all)] + + for i in 1:length(modellists) + #i = 1 + modellist = modellists[i] + status_tuple = status_tuples[i] + outs_vector = outs_vectors[i] + all_vars = init_variables(modellist) + for j in 1:length(meteos) + meteo = meteos[j] + for k in 1:length(outs_vector) + #k = 1 + out_tuple = outs_vector[k] + + try out_st, out_mt = run_single_and_multi_thread_modellist(modellist, out_tuple, meteo) + @test compare_outputs_modellists(out_st, out_mt) + catch e + #print(i," ", j, " ", k) + #println() + if isa(e, DimensionMismatch) + continue + elseif isa(e, ErrorException) + showerror(stdout, e) + @test false + else + showerror(stdout, e) + @test false + end + end + end + end + end +end \ No newline at end of file diff --git a/test/test-simulation.jl b/test/test-simulation.jl index 2ca13f13b..4ff83a64d 100644 --- a/test/test-simulation.jl +++ b/test/test-simulation.jl @@ -28,10 +28,10 @@ end; Process1Model(1.0), status=(var1=15.0, var2=0.3) ) - run!(models) + outputs = run!(models) - vars = keys(status(models)) - @test [models[i][1] for i in vars] == [15.0, 0.3, 5.5] + vars = keys(outputs) + @test [outputs[i][1] for i in vars] == [15.0, 0.3, 5.5] end; @@ -47,12 +47,12 @@ end; meteo = Atmosphere(T=20.0, Wind=1.0, Rh=0.65) - run!(models, meteo) - vars = keys(status(models)) - @test [models[i][1] for i in vars] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] + modellist_outputs = run!(models, meteo) + vars = keys(modellist_outputs) + @test [modellist_outputs[i][1] for i in vars] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] - mtg, mapping, out = check_multiscale_simulation_is_equivalent_begin(models, status_nt, Weather([meteo])) - @test check_multiscale_simulation_is_equivalent_end(models, mtg, mapping, out, Weather([meteo])) + mtg, mapping, out = check_multiscale_simulation_is_equivalent_begin(models, status_nt, meteo) + @test check_multiscale_simulation_is_equivalent_end(modellist_outputs, mtg, mapping, out, meteo) end; @testset "Simulation: 1 time-step, 1 Atmosphere, 2 objects" begin @@ -73,15 +73,15 @@ end; meteo = Atmosphere(T=20.0, Wind=1.0, Rh=0.65) @testset "simulation with an array of objects" begin - run!([models, models2], meteo) - @test [models[i][1] for i in keys(status(models))] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] - @test [models2[i][1] for i in keys(status(models2))] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] + outputs_vector = run!([models, models2], meteo) + @test [outputs_vector[1][i][1] for i in keys(outputs_vector[1])] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] + @test [outputs_vector[2][i][1] for i in keys(outputs_vector[2])] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] end @testset "simulation with a dict of objects" begin - run!(Dict("mod1" => models, "mod2" => models2), meteo) - @test [models[i][1] for i in keys(status(models))] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] - @test [models2[i][1] for i in keys(status(models2))] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] + outputs_vector = run!(Dict("mod1" => models, "mod2" => models2), meteo) + @test [outputs_vector["mod1"][1][i] for i in keys(outputs_vector["mod1"])] == [34.95, 22.0, 56.95, 15.0, 5.5, 0.3] + @test [outputs_vector["mod2"][1][i] for i in keys(outputs_vector["mod2"])] == [36.95, 26.0, 62.95, 15.0, 6.5, 0.3] end end; @@ -95,9 +95,9 @@ end; meteo = Atmosphere(T=20.0, Wind=1.0, Rh=0.65) - run!(models, meteo) - vars = keys(status(models)) - @test [models[i] for i in vars] == [ + outputs = run!(models, meteo) + vars = keys(outputs) + @test [outputs[i] for i in vars] == [ [34.95, 35.550000000000004], [22.0, 23.2], [56.95, 58.75], @@ -125,9 +125,9 @@ end; ] ) - run!(models, meteo) - vars = keys(status(models)) - @test [models[i] for i in vars] == [ + modellist_outputs = run!(models, meteo) + vars = keys(modellist_outputs) + @test [modellist_outputs[i] for i in vars] == [ [34.95, 40.0], [22.0, 23.2], [56.95, 63.2], @@ -137,7 +137,7 @@ end; ] mtg, mapping, out = check_multiscale_simulation_is_equivalent_begin(models, status_nt, meteo) - @test check_multiscale_simulation_is_equivalent_end(models, mtg, mapping, out, meteo) + @test check_multiscale_simulation_is_equivalent_end(modellist_outputs, mtg, mapping, out, meteo) end; @@ -164,21 +164,21 @@ end; ) @testset "simulation with an array of objects" begin - run!([models, models2], meteo) - @test [models[i] for i in keys(status(models))] == [ + outputs_vector = run!([models, models2], meteo) + @test [outputs_vector[1][i] for i in keys(outputs_vector[1])] == [ [34.95, 40.0], [22.0, 23.2], [56.95, 63.2], [15.0, 16.0], [5.5, 5.8], [0.3, 0.3] ] - @test [models2[i] for i in keys(status(models2))] == [ + @test [outputs_vector[2][i] for i in keys(outputs_vector[2])] == [ [36.95, 42.0], [26.0, 27.2], [62.95, 69.2], [15.0, 16.0], [6.5, 6.8], [0.3, 0.3] ] end @testset "simulation with a dict of objects" begin - run!(Dict("mod1" => models, "mod2" => models2), meteo) - @test [models[i] for i in keys(status(models))] == [ + outputs_vector = run!(Dict("mod1" => models, "mod2" => models2), meteo) + @test [[outputs_vector["mod1"][1][i], outputs_vector["mod1"][2][i]] for i in keys(outputs_vector["mod1"])] == [ [34.95, 40.0], [22.0, 23.2], [56.95, 63.2], [15.0, 16.0], [5.5, 5.8], [0.3, 0.3] ] - @test [models2[i] for i in keys(status(models2))] == [ + @test [[outputs_vector["mod2"][1][i], outputs_vector["mod2"][2][i]] for i in keys(outputs_vector["mod2"])] == [ [36.95, 42.0], [26.0, 27.2], [62.95, 69.2], [15.0, 16.0], [6.5, 6.8], [0.3, 0.3] ] end @@ -211,10 +211,85 @@ end; leaf[:var1] = 15.0 - out = @test_nowarn run!(mtg, mapping, meteo) + #out = @test_nowarn run!(mtg, mapping, meteo) + nsteps = PlantSimEngine.get_nsteps(meteo) + sim = PlantSimEngine.GraphSimulation(mtg, mapping, nsteps=nsteps, check=true) + out = @test_nowarn run!(sim,meteo) vars = (:var4, :var6, :var5, :var1, :var2, :var3) - @test [out.statuses["Leaf"][1][i] for i in vars] == [ + @test [sim.statuses["Leaf"][1][i] for i in vars] == [ 22.0, 61.4, 39.4, 15.0, 0.3, 5.5 ] end; + + +@testset "Meteo+ModelList/mapping+outputs combos either valid or different status vector size vs meteo length either run successfully or return a DimensionMisMatch" begin + + meteos = get_simple_meteo_bank() + modellists, status_tuples, outputs_tuples_vectors = get_modellist_bank() + + for i in 1:length(modellists) +# i = 3 + modellist = modellists[i] + status_tuple = status_tuples[i] + outs_vector = outputs_tuples_vectors[i] + + for j in 1:length(meteos) +# j = 1 + meteo = meteos[j] + for k in 1:length(outs_vector) +# k = 7 + out_tuple = outs_vector[k] + @test try outs_modellist = run!(modellist, meteo; tracked_outputs=out_tuple) + true + catch e + print(i," ", j, " ", k) + println() + if isa(e, DimensionMismatch) + true + elseif isa(e, ErrorException) + showerror(stdout, e) + false + else + showerror(stdout, e) + false + end + end + end + end + end + + mtgs, mappings, outs_tuples_vectors_mappings = get_simple_mapping_bank() + + for i in 1:length(mappings) +# i = 1 + mapping = mappings[i] + outs_vector = outs_tuples_vectors_mappings[i] + + for j in 1:length(meteos) +# j = 1 + meteo = meteos[j] + for k in 1:length(outs_vector) +# k = 4 + out_tuple = outs_vector[k] + + mtg = deepcopy(mtgs[i]) + try + outs_multiscale = run!(mtg, mapping, meteo; tracked_outputs=out_tuple) + @test true + catch e + print(i," ", j, " ", k) + println() + if isa(e, DimensionMismatch) + @test true + #elseif isa(e, ErrorException) + else + #@enter outs_multiscale = run!(mtg, mapping, meteo; tracked_outputs=out_tuple) + showerror(stdout, e) + @test false + end + end + end + end + end +end \ No newline at end of file diff --git a/test/test-toy_models.jl b/test/test-toy_models.jl index 3912e1280..7a4fd1354 100644 --- a/test/test-toy_models.jl +++ b/test/test-toy_models.jl @@ -1,5 +1,7 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), DataFrame, header=18) +# Note (smack) : The first test's behaviour is weird to me, because there is an [Info :] that correctly indicates +# :LAI is not initialised, yet @test_nowarn doesn't capture it. I'm not sure what the intended test was, between 'Info' and 'Warn' @testset "ToyLAIModel" begin @test_nowarn ModelList(ToyLAIModel()) @test_nowarn ModelList(ToyLAIModel(), status=(TT_cu=10,)) @@ -13,11 +15,11 @@ meteo_day = CSV.read(joinpath(pkgdir(PlantSimEngine), "examples/meteo_day.csv"), status=(TT_cu=cumsum(meteo_day.TT),), ) - @test_nowarn run!(m) + outputs = @test_nowarn run!(m) @test m[:TT_cu] == cumsum(meteo_day.TT) - @test m[:LAI][begin] ≈ 0.00554987593080316 - @test m[:LAI][end] ≈ 0.0 + @test outputs[:LAI][begin] ≈ 0.00554987593080316 + @test outputs[:LAI][end] ≈ 0.0 end @testset "ToyLAIModel+Beer" begin @@ -27,10 +29,10 @@ end status=(TT_cu=cumsum(meteo_day.TT),), ) - run!(models, meteo_day) + outputs = run!(models, meteo_day) - @test mean(models.status[:aPPFD]) ≈ 9.511021781482347 - @test mean(models.status[:LAI]) ≈ 1.098492557536525 + @test mean(outputs[:aPPFD]) ≈ 9.511021781482347 + @test mean(outputs[:LAI]) ≈ 1.098492557536525 end @@ -45,8 +47,8 @@ end status=(aPPFD=30.0,), ) - run!(model, executor=SequentialEx()) - @test model.status[:biomass] ≈ rue * model.status[:aPPFD] + outputs = run!(model, executor=SequentialEx()) + @test outputs[:biomass][1] ≈ rue * model.status[:aPPFD] # Several time steps: model = ModelList( @@ -54,8 +56,8 @@ end status=(aPPFD=[10.0, 30.0, 25.0],), ) - run!(model, executor=SequentialEx()) - @test model.status[:biomass] ≈ cumsum(rue * model.status[:aPPFD]) + outputs = run!(model, executor=SequentialEx()) + @test outputs[:biomass] ≈ cumsum(rue * model.status[:aPPFD]) end @testset "ToyAssimGrowthModel" begin @@ -73,8 +75,8 @@ end @test to_initialize(model) == NamedTuple() - run!(model) - @test model.status[:biomass] ≈ [4.5] + outputs = run!(model) + @test outputs[:biomass] ≈ [4.5] # Several time steps: model = ModelList( @@ -82,9 +84,9 @@ end status=(aPPFD=[10.0, 30.0, 25.0],), ) - run!(model) - @test model.status[:biomass] ≈ cumsum(model.status[:biomass_increment]) - @test model.status[:biomass_increment] ≈ [0.8333333333333334, 4.5, 3.5833333333333335] + outputs = run!(model) + @test outputs[:biomass] ≈ cumsum(outputs[:biomass_increment]) + @test outputs[:biomass_increment] ≈ [0.8333333333333334, 4.5, 3.5833333333333335] end @testset "ToyLAIModel+Beer+ToyRUEGrowthModel" begin @@ -100,9 +102,9 @@ end @test_logs (:warn, r"A parallel executor was provided") run!(models, meteo_day) # If we provide a serial executor, it works without a warning: - @test_nowarn run!(models, meteo_day, executor=SequentialEx()) + outputs = @test_nowarn run!(models, meteo_day, executor=SequentialEx()) - @test mean(models.status[:aPPFD]) ≈ 9.511021781482347 - @test mean(models.status[:LAI]) ≈ 1.098492557536525 - @test models.status[:biomass][end] ≈ 1041.4687939085675 rtol = 1e-4 + @test mean(outputs[:aPPFD]) ≈ 9.511021781482347 + @test mean(outputs[:LAI]) ≈ 1.098492557536525 + @test outputs[:biomass][end] ≈ 1041.4687939085675 rtol = 1e-4 end \ No newline at end of file