Merge pull request #3 from calliope-project/add-machteld-filtering

irm-codebase · web-flow · commit 539a29202039 · 2026-01-30T17:50:55.000+01:00
Add filtering based on van den Broek et al 2010
diff --git a/config/config.yaml b/config/config.yaml
@@ -4,16 +4,65 @@ crs:
   projected: "epsg:3035"
 
 imputation:
-  bounds_mtco2:
-    co2stop_polygons:
-      # Constrain rows within the CO2Stop database
-      lower: 0
-      upper: 300000
-    aggregated_polygons:
-      # Constrain rows within your requested regions
-      lower: 5
-      upper: 5000000  # TtCO2 -> essentially no upper limit
-  filter:
-    surface_issues: False
-    subsurface_issues: False
-    artificial_polygons: True
+  # Applies after aggregating to the requested shapes.parquet file
+  # Upper of 5 TtCO2 -> essentially no upper limit
+  aggregated:
+    aquifer:
+      bounds_mtco2:
+        lower: 5
+        upper: 5000000
+    gas:
+      bounds_mtco2:
+        lower: 5
+        upper: 5000000
+    oil:
+      bounds_mtco2:
+        lower: 5
+        upper: 5000000
+  # Applies per reservoir polygon within the CO2Stop database
+  co2stop:
+    # minimums/bounds based on van den Broek et al 2010 (Table 2)
+    # Seal thickness and permeability have been set to 0 due to missing data.
+    # https://doi.org/10.1016/j.envsoft.2010.06.015
+    aquifer:
+      bounds_mtco2:
+        lower: 2
+        upper: 300000
+      minimums:
+        porosity_ratio: 0.1
+        depth_m: 800
+        reservoir_thickness_m: 10
+        seal_thickness_m: 0
+        permeability_md: 0
+      remove_w_remarks:
+        surface_issues: False
+        subsurface_issues: False
+        artificial_polygons: True
+    gas:
+      bounds_mtco2:
+        lower: 4
+        upper: 300000
+      minimums:
+        porosity_ratio: 0
+        depth_m: 800
+        reservoir_thickness_m: 10
+        seal_thickness_m: 0
+        permeability_md: 0
+      remove_w_remarks:
+        surface_issues: False
+        subsurface_issues: False
+        artificial_polygons: True
+    oil:
+      bounds_mtco2:
+        lower: 4
+        upper: 300000
+      minimums:
+        porosity_ratio: 0
+        depth_m: 800
+        reservoir_thickness_m: 10
+        seal_thickness_m: 0
+        permeability_md: 0
+      remove_w_remarks:
+        surface_issues: False
+        subsurface_issues: False
+        artificial_polygons: True
diff --git a/docs/README.md b/docs/README.md
@@ -16,9 +16,8 @@ Storage units and traps are removed according to the following.
     - Cases that were not assessed or for which data was undisclosed.
     - Ambiguous duplicates (these are two small traps located in the north sea with repeated IDs and capacities).
     - Optionally, details from the dataset are used to remove the following, if configured:
-        - Cases marked as having surface issues (e.g., a protected area).
-        - Cases marked as having subsurface issues (e.g., ground water).
-        - Artificial polygons created by the CO2Stop authors (i.e., the true spatial extent is unknown).
+        - Qualitative cases marked as having surface or subsurface issues, and artificially created polygons.
+        - Quantitative values (e.g., porosity, depth, ...).
 1. To avoid double-counting, traps within the remaining storage units are removed as their capacity is already included in the storage unit total.
 Please consult the [CO2Stop Final report](https://energy.ec.europa.eu/publications/assessment-co2-storage-potential-europe-co2stop_en) (section 2.3.1) for details.
 1. Three scenarios (`low`, `medium`, `high`) are created for each sequestration type (`aquifer`, `gas`, `oil`) for the remaining CO2Stop data.
@@ -30,10 +29,10 @@ These shapes should follow the schema provided by the [geo-boundaries module](ht
 
 >[!WARNING]
 >Estimates from the CO2Stop dataset are biased by disclosure (or lack thereof), and the filtering settings used.
->Some countries are affected more than others, with Germany being heavily affected by lack of disclosure.
->We provide automated figures so users can evaluate how these aspects affect polygon selection.
+>Some countries are affected more than others, with Germany having particularly poor disclosure.
+>We provide automated figures and logging so users can evaluate how their settings affect polygon selection.
 >
->Below is an example for storage unit aquifers where only undisclosed and artificial polygons have been removed:
+>Below is an example for storage unit aquifers where only undisclosed and artificial polygons have been removed. This can be seen as a _MINIMUM_ amount of removals.
 >![filters](./aquifer_kept.png)
 
 ## Configuration
@@ -46,10 +45,13 @@ See the [interface file](./../INTERFACE.yaml).
 
 ## Data sources
 
-- Poulsen, N., Holloway, S., Neele, F., Smith, N.A., Kirk, K., 2012. CO2StoP Executive Summary (No. ENER/C1/154-2011-SI2.611598). GEOLOGICAL SURVEY OF DENMARK AND GREENLAND.
+- Poulsen, N., Holloway, S., Neele, F., Smith, N.A., Kirk, K., 2012. CO2StoP Executive Summary (No. ENER/C1/154-2011-SI2.611598). GEOLOGICAL SURVEY OF DENMARK AND GREENLAND. <https://energy.ec.europa.eu/publications/assessment-co2-storage-potential-europe-co2stop_en>.
 
 ## References
 
 This module relies on code and methods from the following sources.
 
-- [Ruiz Manuel, I. clio - module_geo_boundaries [Computer software].](https://github.com/calliope-project/module_geo_boundaries/)
+- **Shape schema definition:**
+Ruiz Manuel, I. clio - module_geo_boundaries [Computer software]. <https://github.com/calliope-project/module_geo_boundaries/>.
+- **Filtering minimum defaults:**
+Van Den Broek, M., Brederode, E., Ramírez, A., Kramers, L., Van Der Kuip, M., Wildenborg, T., Turkenburg, W., Faaij, A., 2010. Designing a cost-effective CO2 storage infrastructure using a GIS based linear optimization energy model. Environmental Modelling & Software 25, 1754–1768. <https://doi.org/10.1016/j.envsoft.2010.06.015>.
diff --git a/workflow/internal/config.schema.yaml b/workflow/internal/config.schema.yaml
@@ -11,8 +11,8 @@ $defs:
       - type: string
       - type: integer
 
-  nonNegativeInteger:
-    type: integer
+  nonNegativeNumber:
+    type: number
     minimum: 0
 
   boundsMtco2:
@@ -22,50 +22,60 @@ $defs:
     required: [lower, upper]
     properties:
       lower:
-        $ref: "#/$defs/nonNegativeInteger"
+        $ref: "#/$defs/nonNegativeNumber"
       upper:
-        $ref: "#/$defs/nonNegativeInteger"
+        $ref: "#/$defs/nonNegativeNumber"
 
-properties:
-  crs:
-    description: "CRS values to use during projections."
+  aggregatedConfig:
+    description: "Settings for each aggregated CDR group."
     type: object
     additionalProperties: false
-    required: [geographic, projected]
+    required: [bounds_mtco2]
     properties:
-      geographic:
-        description: "Geographic CRS (for lat/lon operations)."
-        allOf:
-          - $ref: "#/$defs/crsValue"
-      projected:
-        description: "Projected CRS (for area operations)."
+      bounds_mtco2:
+        description: "These bounds will be applied to each shape_id case after aggregation."
         allOf:
-          - $ref: "#/$defs/crsValue"
+          - $ref: "#/$defs/boundsMtco2"
 
-  imputation:
-    description: "These settings alter how data is processed."
+  co2stopConfig:
+    description: "Settings for each CDR group in the CO2Stop dataset."
     type: object
     additionalProperties: false
-    required: [filter, bounds_mtco2]
+    required: [bounds_mtco2, minimums, remove_w_remarks]
     properties:
       bounds_mtco2:
+        description: "These bounds will be applied to each row in the CO2Stop dataset."
+        allOf:
+          - $ref: "#/$defs/boundsMtco2"
+      minimums:
         description: |
-          Lower and upper bounds for the CO2Stop polygons.
-          These are applied before aggregation.
+          Minimum values for key reservoir attributes. Cases below them will be dropped.
+          IMPORTANT: CO2Stop uses '0' for missing values, making NaN and actual 0 indistinguishable.
+          Some of these settings may result in a lot of removed data due to this dataset limitation!
         type: object
-        required: [co2stop_polygons, aggregated_polygons]
+        additionalProperties: false
+        required: [porosity_ratio, depth_m, reservoir_thickness_m, seal_thickness_m, permeability_md]
         properties:
-          co2stop_polygons:
-            description: "These bounds will be applied to each row in the CO2Stop dataset."
-            allOf:
-              - $ref: "#/$defs/boundsMtco2"
-          aggregated_polygons:
-            description: "These bounds will be applied to each shape requested for aggregation."
-            allOf:
-              - $ref: "#/$defs/boundsMtco2"
-      filter:
+          porosity_ratio:
+            description: Minimum mean porosity ratio
+            type: number
+            minimum: 0
+            maximum: 1
+          depth_m:
+            description: Minimum mean depth in metres
+            $ref: "#/$defs/nonNegativeNumber"
+          reservoir_thickness_m:
+            description: Minimum mean reservoir thickness in metres
+            $ref: "#/$defs/nonNegativeNumber"
+          seal_thickness_m:
+            description: Minimum seal thickness in metres
+            $ref: "#/$defs/nonNegativeNumber"
+          permeability_md:
+            description: Permeability in millidarcy (md)
+            $ref: "#/$defs/nonNegativeNumber"
+      remove_w_remarks:
         description: |
-          If enabled, these will remove CO2Stop polygons in certain cases.
+          Remove CO2Stop polygons in certain cases.
           Note that CO2Stop settings and comments are the only thing used for these filters.
           Quality may vary due to the inconsistent nature of the dataset.
         type: object
@@ -83,3 +93,56 @@ properties:
               These are cases where the authors state that polygons do not reflect the actual extent of storage.
               RECOMMENDED TO LEAVE AS 'True'.
             type: boolean
+
+properties:
+  crs:
+    description: "CRS values to use during projections."
+    type: object
+    additionalProperties: false
+    required: [geographic, projected]
+    properties:
+      geographic:
+        description: "Geographic CRS (for lat/lon operations)."
+        allOf:
+          - $ref: "#/$defs/crsValue"
+      projected:
+        description: "Projected CRS (for area operations)."
+        allOf:
+          - $ref: "#/$defs/crsValue"
+
+  imputation:
+    description: "These settings alter how data is processed."
+    type: object
+    additionalProperties: false
+    required: [aggregated, co2stop]
+    properties:
+      aggregated:
+        description: Settings applied after aggregating to the provided shapes file.
+        type: object
+        additionalProperties: false
+        required: [aquifer, gas, oil]
+        properties:
+          aquifer:
+            allOf:
+              - $ref: "#/$defs/aggregatedConfig"
+          gas:
+            allOf:
+              - $ref: "#/$defs/aggregatedConfig"
+          oil:
+            allOf:
+              - $ref: "#/$defs/aggregatedConfig"
+      co2stop:
+        description: Settings applied while preparing / standardising the CO2Stop dataset.
+        type: object
+        additionalProperties: false
+        required: [aquifer, gas, oil]
+        properties:
+          aquifer:
+            allOf:
+              - $ref: "#/$defs/co2stopConfig"
+          gas:
+            allOf:
+              - $ref: "#/$defs/co2stopConfig"
+          oil:
+            allOf:
+              - $ref: "#/$defs/co2stopConfig"
diff --git a/workflow/rules/aggregate.smk b/workflow/rules/aggregate.smk
@@ -5,7 +5,9 @@ rule aggregate_co2stop:
     message:
         "Aggregating '{wildcards.shapes}-{wildcards.scenario}-{wildcards.cdr_group}'."
     params:
-        bounds_mtco2=config["imputation"]["bounds_mtco2"]["aggregated_polygons"],
+        bounds_mtco2=lambda wc: config["imputation"]["aggregated"][wc.cdr_group][
+            "bounds_mtco2"
+        ],
         proj_crs=config["crs"]["projected"],
     input:
         shapes="resources/user/{shapes}/shapes.parquet",
diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk
@@ -7,8 +7,7 @@ rule prepare_co2stop_storage_units:
     message:
         "Harmonising CO2Stop {wildcards.dataset}:{wildcards.cdr_group}."
     params:
-        bounds_mtco2=lambda wc: config["imputation"]["bounds_mtco2"]["co2stop_polygons"],
-        filters=config["imputation"]["filter"],
+        cdr_group_config=lambda wc: config["imputation"]["co2stop"][wc.cdr_group],
         geo_crs=config["crs"]["geographic"],
     input:
         table="resources/automatic/co2stop/storage_table.csv",
@@ -43,8 +42,7 @@ rule prepare_co2stop_traps:
     message:
         "Harmonising CO2Stop {wildcards.dataset}:{wildcards.cdr_group}."
     params:
-        bounds_mtco2=config["imputation"]["bounds_mtco2"]["co2stop_polygons"],
-        filters=config["imputation"]["filter"],
+        cdr_group_config=lambda wc: config["imputation"]["co2stop"][wc.cdr_group],
         geo_crs=config["crs"]["geographic"],
     input:
         table=rules.unzip_co2stop.output.traps_data,
diff --git a/workflow/scripts/prepare_co2stop.py b/workflow/scripts/prepare_co2stop.py
