Skip to content

Commit 5718c8a

Browse files
sjpfenningersjpfenninger
committed
Allow workflow to break larger shapes into subunits; other improvements
* Allow per-subunit tech configuration override * Clean up download rules, download full TIF files by default * Improve plotting
1 parent d18037d commit 5718c8a

15 files changed

Lines changed: 457 additions & 114 deletions

‎config/config.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
1+
tiny_files: False
2+
13
# Options for buffering: either a string of the form "epsg:xxxx" or "UTM"/"utm"
24
# - "UTM": project each shape to the UTM zone of its centroid for buffering
35
# - "epsg:xxxx": use the specified CRS for all buffering
46
# A good option is "epsg:8857" (WGS 84 / Equal Earth Greenwich) for global coverage
57
buffer_crs: "epsg:8857"
68

9+
split_by: country_id
10+
711
land_cover_types:
812
POST_FLOODING: FARM
913
RAINFED_CROPLANDS: FARM
@@ -99,3 +103,17 @@ techs:
99103
protected: 0
100104
shapes_buffer:
101105
land: 10000 # meters
106+
107+
# Optional: override settings for specific subunits (countries, regions, etc.)
108+
# This allows you to set specific parameters that differ from the defaults,
109+
# or apply settings that are not defined in the defaults (defaults = `techs` section).
110+
# The subunit keys should match the subunit IDs used in column selected in `split_by`.
111+
overrides:
112+
PRT: # Inside a subunit, any setting from `techs` can be overridden
113+
wind_offshore:
114+
shapes_buffer:
115+
land: 2000 # meters
116+
NLD:
117+
wind_offshore:
118+
shapes_buffer:
119+
land: 22000 # 22 km = 12 nautical miles

‎tests/integration/test_config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1+
2+
13
module_area_potentials:
4+
tiny_files: True
25
buffer_crs: "epsg:8857"
6+
split_by: country_id
37

48
land_cover_types:
59
POST_FLOODING: FARM

‎workflow/Snakefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,13 @@ workflow.source_path("scripts/geo.py")
2323

2424
wildcard_constraints:
2525
shape="[a-zA-Z0-9_-]+",
26+
subunit="[a-zA-Z0-9_]+",
2627
tech="|".join(config["techs"].keys()),
2728

2829

2930
# Add all your includes here.
31+
include: "rules/functions.smk"
3032
include: "rules/automatic.smk"
31-
include: "rules/prepare.smk"
3233
include: "rules/process.smk"
3334

3435

‎workflow/envs/default.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,5 @@ dependencies:
1919
- pyyaml
2020
- pyproj=3.7.1
2121
- utm=0.7.0
22+
- glom=24.11.0
23+
- dask=2025.7.0

‎workflow/internal/config.schema.yaml‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,17 @@ description: "Schema for user-provided configuration files."
33
type: object
44
additionalProperties: false
55
properties:
6+
tiny_files:
7+
type: boolean
8+
description: "If True, use smaller, clipped files for processing. If False, use full global datasets."
9+
610
buffer_crs:
711
type: string
812
description: "CRS for buffering shapes. Use 'UTM' for UTM zones or 'epsg:xxxx' for a specific CRS."
9-
specs:
10-
type: object
11-
properties:
12-
projection:
13-
type: string
14-
resolution:
15-
type: number
16-
required: [projection, resolution]
17-
additionalProperties: false
13+
14+
split_by:
15+
type: string
16+
description: "Field to split the input shapes by, e.g., 'country_id'."
1817

1918
land_cover_types:
2019
type: object
@@ -48,3 +47,8 @@ properties:
4847
type: object
4948
required: ["initial_area", "continuous_layers", "binary_layers"]
5049
additionalProperties: false
50+
51+
overrides:
52+
type: object
53+
additionalProperties:
54+
type: object

‎workflow/rules/automatic.smk‎

Lines changed: 177 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,117 @@
11
"""Rules to used to download automatic resource files."""
22

3+
if config["tiny_files"]:
34

4-
rule download_cutout_slope:
5-
message:
6-
"Download slope data covering the bounds of the input shapefile."
7-
params:
8-
cog_url=internal["resources"]["automatic"]["slope"],
9-
input:
10-
vector="resources/user/shapes/{shape}.parquet",
11-
output:
12-
path="resources/automatic/cutout/{shape}/slope.tif",
13-
log:
14-
"logs/{shape}/download_cutout_slope.log",
15-
wrapper:
16-
"v7.2.0/geo/rasterio/clip-geotiff"
5+
##
6+
# Directly download clipped slope and bathymetry data
7+
##
178

9+
rule clip_slope:
10+
message:
11+
"Download slope data covering the bounds of the input shapefile."
12+
params:
13+
cog_url=internal["resources"]["automatic"]["slope"],
14+
input:
15+
vector="resources/user/shapes/{shape}.parquet",
16+
output:
17+
path="resources/automatic/cutout/{shape}/slope.tif",
18+
log:
19+
"logs/{shape}/clip_slope.log",
20+
wrapper:
21+
"v7.2.0/geo/rasterio/clip-geotiff"
1822

19-
rule download_cutout_bathymetry:
20-
message:
21-
"Download bathymetry data covering the bounds of the input shapefile."
22-
params:
23-
cog_url=internal["resources"]["automatic"]["bathymetry"],
24-
input:
25-
vector="resources/user/shapes/{shape}.parquet",
26-
output:
27-
path="resources/automatic/cutout/{shape}/bathymetry.tif",
28-
log:
29-
"logs/{shape}/download_cutout_bathymetry.log",
30-
wrapper:
31-
"v7.2.0/geo/rasterio/clip-geotiff"
23+
rule clip_bathymetry:
24+
message:
25+
"Download bathymetry data covering the bounds of the input shapefile."
26+
params:
27+
cog_url=internal["resources"]["automatic"]["bathymetry"],
28+
input:
29+
vector="resources/user/shapes/{shape}.parquet",
30+
output:
31+
path="resources/automatic/cutout/{shape}/bathymetry.tif",
32+
log:
33+
"logs/{shape}/clip_bathymetry.log",
34+
wrapper:
35+
"v7.2.0/geo/rasterio/clip-geotiff"
36+
37+
else:
38+
39+
##
40+
# Download global slope and bathymetry data, then clip the files locally
41+
##
42+
43+
rule download_slope:
44+
message:
45+
"Download global slope data."
46+
params:
47+
url=internal["resources"]["automatic"]["slope"],
48+
output:
49+
path="resources/automatic/global/slope.tif",
50+
log:
51+
"logs/download_slope.log",
52+
conda:
53+
"../envs/shell.yaml"
54+
shell:
55+
"""
56+
curl -sSLo "{output}" "{params.url}"
57+
"""
58+
59+
rule download_bathymetry:
60+
message:
61+
"Download global bathymetry data."
62+
params:
63+
url=internal["resources"]["automatic"]["bathymetry"],
64+
output:
65+
path="resources/automatic/global/bathymetry.tif",
66+
log:
67+
"logs/download_bathymetry.log",
68+
conda:
69+
"../envs/shell.yaml"
70+
shell:
71+
"""
72+
curl -sSLo "{output}" "{params.url}"
73+
"""
74+
75+
rule clip_slope:
76+
message:
77+
"Cut slope data to the bounds of the input shapefile."
78+
input:
79+
script=workflow.source_path("../scripts/clip_raster.py"),
80+
shapes="resources/user/shapes/{shape}.parquet",
81+
slope=rules.download_slope.output,
82+
output:
83+
"resources/automatic/cutout/{shape}/slope.tif",
84+
log:
85+
"logs/{shape}/clip_slope.log",
86+
conda:
87+
"../envs/default.yaml"
88+
shell:
89+
"""
90+
python "{input.script}" "{input.slope}" "{input.shapes}" "{output}" 2> "{log}"
91+
"""
92+
93+
rule clip_bathymetry:
94+
message:
95+
"Cut bathymetry data to the bounds of the input shapefile."
96+
input:
97+
script=workflow.source_path("../scripts/clip_raster.py"),
98+
shapes="resources/user/shapes/{shape}.parquet",
99+
bathymetry=rules.download_bathymetry.output,
100+
output:
101+
"resources/automatic/cutout/{shape}/bathymetry.tif",
102+
log:
103+
"logs/{shape}/clip_bathymetry.log",
104+
conda:
105+
"../envs/default.yaml"
106+
shell:
107+
"""
108+
python "{input.script}" "{input.bathymetry}" "{input.shapes}" "{output}" 2> "{log}"
109+
"""
110+
111+
112+
##
113+
# Globcover
114+
##
32115

33116

34117
rule download_globcover:
@@ -68,6 +151,30 @@ rule unzip_globcover:
68151
"""
69152

70153

154+
rule clip_landcover:
155+
message:
156+
"Cut land cover data to the bounds of the input shapefile."
157+
input:
158+
script=workflow.source_path("../scripts/clip_raster.py"),
159+
shapes="resources/user/shapes/{shape}.parquet",
160+
landcover=rules.unzip_globcover.output,
161+
output:
162+
"resources/automatic/cutout/{shape}/landcover.tif",
163+
log:
164+
"logs/{shape}/clip_landcover.log",
165+
conda:
166+
"../envs/default.yaml"
167+
shell:
168+
"""
169+
python "{input.script}" "{input.landcover}" "{input.shapes}" "{output}" 2> "{log}"
170+
"""
171+
172+
173+
##
174+
# Global Human Settlement Layer (GHSL)
175+
##
176+
177+
71178
rule download_ghsl:
72179
message:
73180
"Download the GHSL (Global Human Settlement Layer) built-up surface data."
@@ -103,3 +210,47 @@ rule unzip_ghsl:
103210
"""
104211
python "{input.script}" "{input.zipfile}" -f "{params.target_file}" -o "{output}" 2> "{log}"
105212
"""
213+
214+
215+
rule clip_settlement:
216+
message:
217+
"Cut settlement data to the bounds of the input shapefile."
218+
input:
219+
script=workflow.source_path("../scripts/clip_raster.py"),
220+
shapes="resources/user/shapes/{shape}.parquet",
221+
settlement=rules.unzip_ghsl.output,
222+
output:
223+
"resources/automatic/cutout/{shape}/settlement.tif",
224+
log:
225+
"logs/{shape}/clip_settlement.log",
226+
conda:
227+
"../envs/default.yaml"
228+
shell:
229+
"""
230+
python "{input.script}" "{input.settlement}" "{input.shapes}" "{output}" 2> "{log}"
231+
"""
232+
233+
234+
##
235+
# Protected Areas (WDPA)
236+
##
237+
238+
239+
rule rasterise_clip_wdpa:
240+
message:
241+
"Rasterise and cut WDPA data to the bounds of the input shapefile, using the landcover raster as reference for the rasterisation."
242+
input:
243+
script=workflow.source_path("../scripts/clip_and_rasterise_polys.py"),
244+
shapes="resources/user/shapes/{shape}.parquet",
245+
reference_raster=rules.clip_landcover.output,
246+
protected_areas="resources/user/wdpa.gdb",
247+
output:
248+
"resources/automatic/cutout/{shape}/wdpa.tif",
249+
log:
250+
"logs/{shape}/clip_wdpa.log",
251+
conda:
252+
"../envs/default.yaml"
253+
shell:
254+
"""
255+
python "{input.script}" "{input.shapes}" "{input.reference_raster}" "{input.protected_areas}" "{output}" 2> "{log}"
256+
"""

‎workflow/rules/functions.smk‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
def get_subunits(wildcards):
2+
checkpoint_output = checkpoints.breakup_shape.get(**wildcards).output[0]
3+
return expand(
4+
"results/{{shape}}/{subunit}/area_potential_{{tech}}.tif",
5+
subunit=glob_wildcards(
6+
os.path.join(checkpoint_output, "{subunit}.parquet")
7+
).subunit,
8+
)

‎workflow/rules/prepare.smk‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

0 commit comments

Comments
 (0)