diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ffd0b8..182555f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: # Python file formatting - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.4 + rev: v0.15.9 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -33,13 +33,13 @@ repos: # Snakemake file formatting - repo: https://github.com/snakemake/snakefmt - rev: v0.11.4 + rev: v1.0.0 hooks: - id: snakefmt # Spelling - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell files: .*\.(py|smk|md)$|^Snakefile$ diff --git a/tests/integration/Snakefile b/tests/integration/Snakefile index 0c0c2d2..c9d9ee4 100644 --- a/tests/integration/Snakefile +++ b/tests/integration/Snakefile @@ -7,10 +7,6 @@ configfile: workflow.source_path("./test_config.yaml") # `config`: specifies the module configuration. # `pathvars:` helps you re-wire where the module places files. module module_hydropower: - snakefile: - "../../workflow/Snakefile" - config: - config["module_hydropower"] pathvars: # Redirect specific user resources (inputs) shapes="resources/inputs/{shapes}/shapes.parquet", @@ -21,6 +17,10 @@ module module_hydropower: logs="resources/hydropower/logs", resources="resources/hydropower/resources", results="resources/hydropower/results", + snakefile: + "../../workflow/Snakefile" + config: + config["module_hydropower"] # rename all module rules with a prefix, to avoid naming conflicts. @@ -33,8 +33,6 @@ SHAPES = ["MNE"] # Request something from the module rule all: - message: - "Generate PU timeseries for a small country." default_target: True input: expand( @@ -42,3 +40,5 @@ rule all: shapes=SHAPES, plant_type=PLANTS, ), + message: + "Generate PU timeseries for a small country." diff --git a/workflow/Snakefile b/workflow/Snakefile index 179d086..7ee941b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -47,8 +47,6 @@ validate_year_config() rule all: - message: - "ERROR: Invalid `rule all:` call" default_target: True output: "INVALID", @@ -56,5 +54,7 @@ rule all: stderr="/all.stderr", conda: "envs/shell.yaml" + message: + "ERROR: Invalid `rule all:` call" shell: 'echo "This workflow must be called as a snakemake module." > {log.stderr}' diff --git a/workflow/rules/automatic.smk b/workflow/rules/automatic.smk index 4bfbac8..427c86b 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/automatic.smk @@ -2,10 +2,6 @@ rule download_eia: - message: - "Download the EIA International energy statistics in bulk." - params: - url=internal["resources"]["automatic"]["EIA"], output: zipfile="/automatic/eia/EIA-INTL.zip", log: @@ -13,37 +9,35 @@ rule download_eia: localrule: True conda: "../envs/shell.yaml" + params: + url=internal["resources"]["automatic"]["EIA"], + message: + "Download the EIA International energy statistics in bulk." shell: r'curl -fsSLo {output.zipfile:q} "{params.url}"' rule download_basin: - message: - "Downloading HydroBASINS file for '{wildcards.continent}'." - params: - url=lambda wc: internal["resources"]["automatic"]["HydroBASINS"].format( - continent=wc.continent - ), output: path="/automatic/hydrobasins/{continent}.zip", + log: + "/download_basin_{continent}.log", wildcard_constraints: continent="|".join(internal["continent_codes"]), + localrule: True conda: "../envs/shell.yaml" - log: - "/download_basin_{continent}.log", - localrule: True + params: + url=lambda wc: internal["resources"]["automatic"]["HydroBASINS"].format( + continent=wc.continent + ), + message: + "Downloading HydroBASINS file for '{wildcards.continent}'." shell: r'curl -fsSLo {output.path:q} "{params.url}"' rule download_cutout: - message: - "Downloading runoff cutout from {params.start_year}-01-01 to {params.end_year}-12-31." - params: - era5_crs=internal["era5_crs"], - start_year=config["years"]["start"], - end_year=config["years"]["end"], input: shapes="", output: @@ -53,10 +47,16 @@ rule download_cutout: caption="../report/cutout.rst", category="Hydropower module", ), - conda: - "../envs/hydropower.yaml" log: "/{shapes}/download_cutout.log", localrule: True + conda: + "../envs/hydropower.yaml" + params: + era5_crs=internal["era5_crs"], + start_year=config["years"]["start"], + end_year=config["years"]["end"], + message: + "Downloading runoff cutout from {params.start_year}-01-01 to {params.end_year}-12-31." script: "../scripts/download_cutout.py" diff --git a/workflow/rules/basins.smk b/workflow/rules/basins.smk index 90dfa7b..99c2fb1 100644 --- a/workflow/rules/basins.smk +++ b/workflow/rules/basins.smk @@ -2,31 +2,29 @@ rule basins_extract_pfafstetter_level: - message: - "Unzipping HydroBASINS file for '{wildcards.continent}' at Pfafstetter level '{params.level}'." - params: - level=lambda wc: wc.level, - continent=lambda wc: wc.continent, input: zip_file=rules.download_basin.output.path, output: parquet_file=temp( "/automatic/hydrobasins/{continent}_{level}.parquet" ), + log: + "/basins_extract_pfafstetter_level_{continent}_{level}.log", wildcard_constraints: continent="|".join(internal["continent_codes"]), level="|".join(internal["pfafstetter_level_codes"]), conda: "../envs/hydropower.yaml" - log: - "/basins_extract_pfafstetter_level_{continent}_{level}.log", + params: + level=lambda wc: wc.level, + continent=lambda wc: wc.continent, + message: + "Unzipping HydroBASINS file for '{wildcards.continent}' at Pfafstetter level '{params.level}'." script: "../scripts/basins_extract_pfafstetter_level.py" rule basins_combine_continents: - message: - "Combine all HydroBASINS into a single dataset for Pfafstetter level '{wildcards.level}'." input: continent_files=expand( "/automatic/hydrobasins/{continent}_{{level}}.parquet", @@ -39,9 +37,11 @@ rule basins_combine_continents: caption="../report/basins.rst", category="Hydropower module", ), - conda: - "../envs/hydropower.yaml" log: "/basins_combine_continents_{level}.log", + conda: + "../envs/hydropower.yaml" + message: + "Combine all HydroBASINS into a single dataset for Pfafstetter level '{wildcards.level}'." script: "../scripts/basins_combine_continents.py" diff --git a/workflow/rules/powerplants.smk b/workflow/rules/powerplants.smk index 21bddb4..4cd2b68 100644 --- a/workflow/rules/powerplants.smk +++ b/workflow/rules/powerplants.smk @@ -2,11 +2,6 @@ rule powerplants_adjust_location: - message: - "Adjusting hydro powerplant location to the nearest shape and basin." - params: - crs=config["crs"], - basin_adjustment=config["powerplants"]["basin_adjustment"], input: basins=f"/automatic/hydrobasins/global_{config["pfafstetter_level"]}.parquet", powerplants="", @@ -22,15 +17,16 @@ rule powerplants_adjust_location: "/{shapes}/powerplants_adjust_location.log", conda: "../envs/hydropower.yaml" + params: + crs=config["crs"], + basin_adjustment=config["powerplants"]["basin_adjustment"], + message: + "Adjusting hydro powerplant location to the nearest shape and basin." script: "../scripts/powerplants_adjust_location.py" rule powerplants_get_inflow_m3: - message: - "Calculating hydro powerplant inflow in m3." - params: - smoothing_hours=config["smoothing_hours"], input: adjusted_powerplants=rules.powerplants_adjust_location.output.adjusted_powerplants, basins=f"/automatic/hydrobasins/global_{config["pfafstetter_level"]}.parquet", @@ -42,16 +38,15 @@ rule powerplants_get_inflow_m3: "/{shapes}/powerplants_get_inflow_m3.log", conda: "../envs/hydropower.yaml" + params: + smoothing_hours=config["smoothing_hours"], + message: + "Calculating hydro powerplant inflow in m3." script: "../scripts/powerplants_get_inflow_m3.py" rule powerplants_get_inflow_mwh: - message: - "Calculating powerplant generation in MWh and applying corrections using historical data." - params: - pu_factor_range=internal["pu_factor_range"], - technology_mapping=config["powerplants"]["technology_mapping"], input: inflow_m3=rules.powerplants_get_inflow_m3.output.inflow, adjusted_powerplants=rules.powerplants_adjust_location.output.adjusted_powerplants, @@ -62,15 +57,16 @@ rule powerplants_get_inflow_mwh: "/{shapes}/powerplants_get_inflow_mwh.log", conda: "../envs/hydropower.yaml" + params: + pu_factor_range=internal["pu_factor_range"], + technology_mapping=config["powerplants"]["technology_mapping"], + message: + "Calculating powerplant generation in MWh and applying corrections using historical data." script: "../scripts/powerplants_get_inflow_mwh.py" rule powerplants_get_pu_per_shape: - message: - "Calculating aggregated per-unit timeseries per shape for '{wildcards.plant_type}'." - params: - technology_mapping=config["powerplants"]["technology_mapping"], input: adjusted_powerplants=rules.powerplants_adjust_location.output.adjusted_powerplants, inflow_mwh="", @@ -81,11 +77,15 @@ rule powerplants_get_pu_per_shape: caption="../report/pu_per_shape.rst", category="Hydropower module", ), - wildcard_constraints: - plant_type="|".join(["run_of_river", "reservoir"]), log: "/{shapes}/powerplants_get_pu_per_shape_{plant_type}.log", + wildcard_constraints: + plant_type="|".join(["run_of_river", "reservoir"]), conda: "../envs/hydropower.yaml" + params: + technology_mapping=config["powerplants"]["technology_mapping"], + message: + "Calculating aggregated per-unit timeseries per shape for '{wildcards.plant_type}'." script: "../scripts/powerplants_get_pu_per_shape.py" diff --git a/workflow/rules/statistics.smk b/workflow/rules/statistics.smk index d2770cd..adcc8e3 100644 --- a/workflow/rules/statistics.smk +++ b/workflow/rules/statistics.smk @@ -2,10 +2,6 @@ rule prepare_statistics: - message: - "Get EIA annual country hydropower generation statistics." - params: - years=config["years"], input: shapes="", eia_bulk=rules.download_eia.output.zipfile, @@ -16,5 +12,9 @@ rule prepare_statistics: "/{shapes}/prepare_statistics.log", conda: "../envs/hydropower.yaml" + params: + years=config["years"], + message: + "Get EIA annual country hydropower generation statistics." script: "../scripts/prepare_statistics.py"