Merge pull request #5 from calliope-project/feature-dynamic-inputs

Feature dynamic inputs

Author: irm-codebase

config/config.yaml

@@ -22,19 +22,19 @@ imputation:
   # Applies per reservoir polygon within the CO2Stop database
   co2stop:
     # minimums/bounds based on van den Broek et al 2010 (Table 2)
-    # Seal thickness and permeability have been set to 0 due to missing data.
+    # Seal thickness, porosity, and permeability have been set to 0 due to missing data.
     # https://doi.org/10.1016/j.envsoft.2010.06.015
     aquifer:
       bounds_mtco2:
         lower: 2
         upper: 300000
       minimums:
-        porosity_ratio: 0.1
+        porosity_ratio: 0
         depth_m: 800
         reservoir_thickness_m: 10
         seal_thickness_m: 0
         permeability_md: 0
-      remove_w_remarks:
+      remove_remarks:
         surface_issues: False
         subsurface_issues: False
         artificial_polygons: True
@@ -48,7 +48,7 @@ imputation:
         reservoir_thickness_m: 10
         seal_thickness_m: 0
         permeability_md: 0
-      remove_w_remarks:
+      remove_remarks:
         surface_issues: False
         subsurface_issues: False
         artificial_polygons: True
@@ -62,7 +62,7 @@ imputation:
         reservoir_thickness_m: 10
         seal_thickness_m: 0
         permeability_md: 0
-      remove_w_remarks:
+      remove_remarks:
         surface_issues: False
         subsurface_issues: False
         artificial_polygons: True

docs/README.md

@@ -30,8 +30,10 @@ These shapes should follow the schema provided by the [geo-boundaries module](ht
 >[!WARNING]
 >Estimates from the CO2Stop dataset are biased by disclosure (or lack thereof), and the filtering settings used.
 >Some countries are affected more than others, with Germany having particularly poor disclosure.
->We provide automated figures and logging so users can evaluate how their settings affect polygon selection.
 >
+>Similarly, CO2Stop suffers from poor data handling practices that make unavailable data and 'true' zero values indistinguishable from each other, amplifying the uneven assignation of sequestration. For example: setting `porosity_ratio: 0.1` will completely remove France in most cases.
+>
+>We provide automated figures and logging (in `logs/storage_units/` and `logs/traps/`) so users can evaluate how their settings affect polygon selection.
 >Below is an example for storage unit aquifers where only undisclosed and artificial polygons have been removed. This can be seen as a _MINIMUM_ amount of removals.
 >![filters](./aquifer_kept.png)
 

workflow/internal/config.schema.yaml

@@ -41,7 +41,7 @@ $defs:
     description: "Settings for each CDR group in the CO2Stop dataset."
     type: object
     additionalProperties: false
-    required: [bounds_mtco2, minimums, remove_w_remarks]
+    required: [bounds_mtco2, minimums, remove_remarks]
     properties:
       bounds_mtco2:
         description: "These bounds will be applied to each row in the CO2Stop dataset."
@@ -73,9 +73,9 @@ $defs:
           permeability_md:
             description: Permeability in millidarcy (md)
             $ref: "#/$defs/nonNegativeNumber"
-      remove_w_remarks:
+      remove_remarks:
         description: |
-          Remove CO2Stop polygons in certain cases.
+          Remove CO2Stop polygons based on remarks by the dataset authors.
           Note that CO2Stop settings and comments are the only thing used for these filters.
           Quality may vary due to the inconsistent nature of the dataset.
         type: object

workflow/rules/aggregate.smk

@@ -11,8 +11,8 @@ rule aggregate_co2stop:
         proj_crs=config["crs"]["projected"],
     input:
         shapes="resources/user/{shapes}/shapes.parquet",
-        storage_units="resources/automatic/co2stop/storage_units/aquifer.parquet",
-        traps="resources/automatic/co2stop/traps/{cdr_group}.parquet",
+        storage_units=rules.prepare_co2stop_storage_units.output.mtco2,
+        traps=rules.prepare_co2stop_traps.output.mtco2,
     output:
         aggregated="results/{shapes}/{scenario}/{cdr_group}.parquet",
         plot=report(
@@ -39,8 +39,11 @@ rule aggregate_totals:
         proj_crs=config["crs"]["projected"],
     input:
         shapes="resources/user/{shapes}/shapes.parquet",
-        aggregates=expand(
-            "results/{{shapes}}/{{scenario}}/{cdr_group}.parquet", cdr_group=CDR_GROUP
+        aggregates=lambda wc: expand(
+            rules.aggregate_co2stop.output.aggregated,
+            shapes=wc.shapes,
+            scenario=wc.scenario,
+            cdr_group=CDR_GROUP,
         ),
     output:
         totals="results/{shapes}/{scenario}/totals.parquet",

workflow/rules/automatic.smk

@@ -28,9 +28,9 @@ rule unzip_co2stop:
     input:
         zipfile=rules.download_co2stop.output.zipfile,
     output:
-        storage_data="resources/automatic/co2stop/storage_table.csv",
+        storage_data="resources/automatic/co2stop/storage_data.csv",
         storage_map="resources/automatic/co2stop/storage_map.kml",
-        traps_data="resources/automatic/co2stop/traps_table1.csv",
+        traps_data="resources/automatic/co2stop/traps_data.csv",
         traps_map="resources/automatic/co2stop/traps_map.kml",
         country_map="resources/automatic/co2stop/countries.kml",
     log:

workflow/rules/prepare.smk

@@ -5,33 +5,32 @@ CDR_GROUP = ["aquifer", "gas", "oil"]
 
 rule prepare_co2stop_storage_units:
     message:
-        "Harmonising CO2Stop {wildcards.dataset}:{wildcards.cdr_group}."
+        "Harmonising CO2Stop storage units: aquifer."
     params:
-        cdr_group_config=lambda wc: config["imputation"]["co2stop"][wc.cdr_group],
+        cdr_group="aquifer",
+        cdr_group_config=lambda wc: config["imputation"]["co2stop"]["aquifer"],
+        dataset="storage_units",
         geo_crs=config["crs"]["geographic"],
     input:
-        table="resources/automatic/co2stop/storage_table.csv",
-        polygons="resources/automatic/co2stop/storage_map.kml",
-        countries="resources/automatic/co2stop/countries.kml",
+        table=rules.unzip_co2stop.output.storage_data,
+        polygons=rules.unzip_co2stop.output.storage_map,
+        countries=rules.unzip_co2stop.output.country_map,
     output:
-        mtco2="resources/automatic/co2stop/{dataset}/{cdr_group}.parquet",
+        mtco2="resources/automatic/co2stop/storage_units/aquifer.parquet",
         plot_kept=report(
-            "resources/automatic/co2stop/{dataset}/{cdr_group}_kept.png",
+            "resources/automatic/co2stop/storage_units/aquifer_kept.png",
             caption="../report/prepare_co2stop_kept.rst",
             category="CO2Stop module",
             subcategory="kept polygons",
         ),
         plot_scenarios=report(
-            "resources/automatic/co2stop/{dataset}/{cdr_group}_scenarios.png",
+            "resources/automatic/co2stop/storage_units/aquifer_scenarios.png",
             caption="../report/prepare_co2stop_scenarios.rst",
             category="CO2Stop module",
             subcategory="scenarios",
         ),
     log:
-        "logs/{dataset}/{cdr_group}/prepare_co2stop.log",
-    wildcard_constraints:
-        dataset="storage_units",
-        cdr_group="aquifer",
+        "logs/storage_units/aquifer/prepare_co2stop.log",
     conda:
         "../envs/co2stop.yaml"
     script:
@@ -40,32 +39,33 @@ rule prepare_co2stop_storage_units:
 
 rule prepare_co2stop_traps:
     message:
-        "Harmonising CO2Stop {wildcards.dataset}:{wildcards.cdr_group}."
+        "Harmonising CO2Stop traps: {wildcards.cdr_group}."
     params:
+        cdr_group=lambda wc: wc.cdr_group,
         cdr_group_config=lambda wc: config["imputation"]["co2stop"][wc.cdr_group],
+        dataset="traps",
         geo_crs=config["crs"]["geographic"],
     input:
         table=rules.unzip_co2stop.output.traps_data,
         polygons=rules.unzip_co2stop.output.traps_map,
         countries=rules.unzip_co2stop.output.country_map,
     output:
-        mtco2="resources/automatic/co2stop/{dataset}/{cdr_group}.parquet",
+        mtco2="resources/automatic/co2stop/traps/{cdr_group}.parquet",
         plot_kept=report(
-            "resources/automatic/co2stop/{dataset}/{cdr_group}_kept.png",
+            "resources/automatic/co2stop/traps/{cdr_group}_kept.png",
             caption="../report/prepare_co2stop_kept.rst",
             category="CO2Stop module",
             subcategory="kept polygons",
         ),
         plot_scenarios=report(
-            "resources/automatic/co2stop/{dataset}/{cdr_group}_scenarios.png",
+            "resources/automatic/co2stop/traps/{cdr_group}_scenarios.png",
             caption="../report/prepare_co2stop_scenarios.rst",
             category="CO2Stop module",
             subcategory="scenarios",
         ),
     log:
-        "logs/{dataset}/{cdr_group}/prepare_co2stop.log",
+        "logs/traps/{cdr_group}/prepare_co2stop.log",
     wildcard_constraints:
-        dataset="traps",
         cdr_group="|".join(CDR_GROUP),
     conda:
         "../envs/co2stop.yaml"

workflow/scripts/prepare_co2stop.py

@@ -306,8 +306,8 @@ def main() -> None:
     if not CRS.from_user_input(geo_crs).is_geographic:
         raise ValueError(f"Expected geographic CRS, got {geo_crs!r}.")
 
-    dataset_name = snakemake.wildcards.dataset
-    cdr_group = snakemake.wildcards.cdr_group
+    dataset_name = snakemake.params.dataset
+    cdr_group = snakemake.params.cdr_group
     config = snakemake.params.cdr_group_config
 
     match dataset_name:
@@ -330,7 +330,7 @@ def main() -> None:
 
     # Identify and remove 'bad apples', depending on the configuration.
     mask_issues = identify_removals(
-        dataset, config["remove_w_remarks"], config["minimums"], data_id
+        dataset, config["remove_remarks"], config["minimums"], data_id
     )
     dataset = dataset[~mask_issues]