Feature: standardisation with powerplants (#14)

* produce the CDS api key within the integration test

* Use CDS envvar in integration tests

* Standardise with future powerplants module

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: irm-codebase

.github/workflows/pr-ci.yml

@@ -6,4 +6,6 @@ on:
 
 jobs:
   ci-tests:
-    uses: calliope-project/data-module-template/.github/workflows/template-pr-ci.yml@latest
+    uses: calliope-project/data-module-template/.github/workflows/template-pr-ci.yml@feature-ci-cdsapi
+    secrets:
+      CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }}

.gitignore

@@ -11,6 +11,9 @@
 __pycache__
 *.pyc
 
+### Environments
+.pixi/
+
 ### Snakemake
 .snakemake/
 gurobi.log
@@ -24,5 +27,3 @@ gurobi.log
 **/results/*
 **/temp/*
 **/tmp/*
-!tests/integration/resources/automatic/
-!tests/integration/resources/user

INTERFACE.yaml

@@ -1,12 +1,17 @@
 # Module Input-Output structure for automated doc. generation
 resources:
   user:
-    national_generation.parquet: Annual hydropower generation statistics per country.
-    powerplants.parquet: Powerplant statistics.
-    shapes.parquet: Resolution shapes, used for capacity aggregation.
+    "{shapes}/powerplants.parquet": Powerplant statistics.
+    "{shapes}/shapes.parquet": Resolution shapes, used for capacity aggregation.
 results:
-  adjusted_powerplants.parquet: >
-    Powerplants statistics after geo-location correction, plus assigned shape_id and country_id.
-    Can be used to identify which powerplants were assigned to a given polygon.
-  by_shape_id/hydro_dam_cf.parquet: Capacity Factors for hydro dams, per shape.
-  by_shape_id/hydro_run_of_rover_cf.parquet: Capacity Factors for run of river plants, per shape.
+  "{shapes}/statistics/generation.parquet": >
+    National hydropower generation statistics for the requested shapes.
+  "{shapes}/disaggregated/inflow_mwh.parquet": >
+    Inflow time series for each powerplant in MWh.
+    Only includes powerplants that fall within the provided shapes after buffering.
+  "{shapes}/aggregated/reservoir_cf.parquet": >
+    Capacity Factors of hydropower reservoirs aggregated to shapes.
+  "{shapes}/aggregated/run_of_river_cf.parquet": >
+    Capacity Factors of run of river plants aggregated to shapes.
+wildcards:
+  shapes: name of the provided shapes case.

README.md

@@ -2,21 +2,8 @@
 
 A module to calculate hydropower inflow timeseries for facilities around the globe. Based on Euro-Calliope methods.
 
-![Hydropower CFs assorted europe](./docs/hydro_dam_cf.png)
-
 A modular `snakemake` workflow built for [`clio`](https://clio.readthedocs.io/) data modules.
 
-## Steps
-
-1. [HydroBasins](https://www.hydrosheds.org/) data is downloaded and combined into a global dataset.
-2. Powerplant positions are geo-corrected to the nearest basin, and then assigned to the nearest shape.
-3. Inflow in cubic meters is obtained for each powerplant location using ERA5 data through the [`atlite`](https://github.com/PyPSA/atlite) library.
-4. Inflow in MWh is obtained for each plant using the given historical national annual hydropower generation and its share of total installed national capacity.
-
-> [!IMPORTANT]
-> The module assumes that provided sample of powerplants sums up to the total national capacity in the given year.
-> Providing incomplete data might lead to distortions in the calculated time series!
-
 ## Using this module
 
 This module can be imported directly into any `snakemake` workflow.
@@ -36,7 +23,7 @@ pixi install --all
 For testing, simply run:
 
 ```shell
-pixi run test
+pixi run test-integration
 ```
 
 To view the documentation locally, use:
@@ -50,5 +37,5 @@ To test a minimal example of a workflow using this module:
 ```shell
 pixi shell    # activate this project's environment
 cd tests/integration/  # navigate to the integration example
-snakemake --use-conda  # run the workflow!
+snakemake --use-conda --cores 2  # run the workflow!
 ```

config/config.yaml

@@ -1,9 +1,12 @@
 # A minimal example of how to configure this module
 pfafstetter_level: "06"
-years:
-  start: 2000
-  end: 2000
+years:  # Include 2011, 2012 data
+  start: 2011
+  end: 2013
 powerplants:
+  technology_mapping:
+    run_of_river: "run of river"
+    reservoir: "reservoir"
   basin_adjustment:
     buffer_radius: 1000  # epsg:3857 is in meters
     max_outside: 50

mypy.ini

@@ -1,2 +1,3 @@
 [mypy]
 disable_error_code = import-untyped
+exclude = (^|/)\.(snakemake|pixi)(/|$)

pixi.lock

@@ -1,41 +1,25 @@
 [project]
 name = "module_hydropower"
 authors = ["See AUTHORS file"]
-description = "A module to calculate hydropower generation timeseries for facilities around the globe."
+description = "A module to calculate hydropower inflow timeseries for facilities around the globe. Based on Euro-Calliope methods."
 license = "Apache-2.0"
 readme = "README.md"
 channels = ["conda-forge", "bioconda"]
 platforms = ["win-64", "linux-64", "osx-arm64"]
 homepage = "https://clio.readthedocs.io/"
 
 [dependencies]
-python = ">=3.12"
-mkdocs-material = ">=9.6.7"
+clio-tools = ">=2025.03.03"
+conda = ">=25.0.0"
+ipdb = ">=0.13.13"
+ipykernel = ">=6.29.5"
+jsonschema = ">=4.0.0"
 mypy = ">=1.15.0"
 pytest = ">=8.3.5"
+python = ">=3.12"
 ruff = ">=0.9.9"
 snakefmt = ">=0.10.2"
-conda = ">=25.0.0"
 snakemake-minimal = ">=8.29.0"
-clio-tools = ">=2025.03.03"
-geopandas = ">=1.0.1,<2"
-ipykernel = ">=6.29.5,<7"
-pyarrow = ">=19.0.1,<20"
-pycountry = ">=24.6.1,<25"
-numpy = ">=2.2.4,<3"
-pandera-geopandas = ">=0.24.0,<0.25"
-contextily = ">=1.6.2,<2"
-ipdb = ">=0.13.13"
-atlite = ">=0.4.0,<0.5"
-geofileops = ">=0.9.2,<0.10"
-xarray = ">=2025.3.0,<2026"
-scipy = ">=1.15.2,<2"
-pandera-io = ">=0.24.0,<0.25"
-
-[pypi-dependencies]
-mkdocs-mermaid2-plugin = ">=1.2.1"
 
 [tasks]
-test = {cmd = "pytest tests/"}
-serve-docs = {cmd = "mkdocs serve"}
-build-docs = {cmd = "mkdocs build"}
+test-integration = {cmd = "pytest tests/clio_test.py"}