Add script path provider (take 2) (PyPSA#1997)

* Add script path getter; update all `script` directives

* force integration test on windows runner

* Revert CI change

---------

Co-authored-by: Lukas Trippe <lkstrp@pm.me>

Author: brynpickering

Snakefile

@@ -18,6 +18,7 @@ from scripts._helpers import (
     get_scenarios,
     get_shadow,
     path_provider,
+    script_path_provider,
 )
 from scripts.lib.validation.config import validate_config
 
@@ -43,6 +44,7 @@ exclude_from_shared = run["shared_resources"]["exclude"]
 logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
 benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
 resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
+scripts = script_path_provider(Path(workflow.snakefile).parent)
 
 RESULTS = "results/" + RDIR
 

doc/release_notes.rst

@@ -9,13 +9,15 @@ Release Notes
 Upcoming Release
 ================
 
+* Add script path getter helper method to allow for rule inheritance in nested snakefiles.
+
 * Include new storage technologies such as li-ion, vanadium, lfp, lair, pair and iron-air. These technologies can now be configured as either store-link combinations or standalone storage units.
   Implemented in both `add_electricity.py` and `prepare_sector_network.py` (https://github.com/PyPSA/pypsa-eur/pull/1961).
 
 * Updated data sources for country-level electricity demand time series. In addition to the OPSD data (``retrieve_electricity_demand_opsd``, demand time series
   are now downloaded via the ENTSO-E Transparency Platform API (``retrieve_electricity_demand_entsoe`` environment variable ``ENTSOE_API_KEY`` required)
   and from the NESO data portal for Great Britain and Northern Ireland (``retrieve_electricity_demand_neso``).
-  Manual corrections and gap filling methods have been applied and checked for data until the end of 2025. 
+  Manual corrections and gap filling methods have been applied and checked for data until the end of 2025.
   (https://github.com/PyPSA/pypsa-eur/pull/1828).
 
 * Applied scaling of kW and GW to MW for custom_costs as well (https://github.com/PyPSA/pypsa-eur/pull/2023).

rules/build_electricity.smk

@@ -29,7 +29,7 @@ rule build_electricity_demand:
     resources:
         mem_mb=5000,
     script:
-        "../scripts/build_electricity_demand.py"
+        scripts("build_electricity_demand.py")
 
 
 rule build_powerplants:
@@ -54,7 +54,7 @@ rule build_powerplants:
     resources:
         mem_mb=7000,
     script:
-        "../scripts/build_powerplants.py"
+        scripts("build_powerplants.py")
 
 
 def input_base_network(w):
@@ -106,7 +106,7 @@ rule base_network:
     resources:
         mem_mb=2000,
     script:
-        "../scripts/base_network.py"
+        scripts("base_network.py")
 
 
 rule build_osm_boundaries:
@@ -123,7 +123,7 @@ rule build_osm_boundaries:
     resources:
         mem_mb=1500,
     script:
-        "../scripts/build_osm_boundaries.py"
+        scripts("build_osm_boundaries.py")
 
 
 rule build_bidding_zones:
@@ -148,7 +148,7 @@ rule build_bidding_zones:
     resources:
         mem_mb=1500,
     script:
-        "../scripts/build_bidding_zones.py"
+        scripts("build_bidding_zones.py")
 
 
 rule build_shapes:
@@ -186,7 +186,7 @@ rule build_shapes:
     resources:
         mem_mb=1500,
     script:
-        "../scripts/build_shapes.py"
+        scripts("build_shapes.py")
 
 
 if CUTOUT_DATASET["source"] in ["build"]:
@@ -206,7 +206,7 @@ if CUTOUT_DATASET["source"] in ["build"]:
         resources:
             mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
         script:
-            "../scripts/build_cutout.py"
+            scripts("build_cutout.py")
 
 
 rule build_ship_raster:
@@ -224,7 +224,7 @@ rule build_ship_raster:
     benchmark:
         benchmarks("build_ship_raster")
     script:
-        "../scripts/build_ship_raster.py"
+        scripts("build_ship_raster.py")
 
 
 rule determine_availability_matrix_MD_UA:
@@ -268,7 +268,7 @@ rule determine_availability_matrix_MD_UA:
     resources:
         mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     script:
-        "../scripts/determine_availability_matrix_MD_UA.py"
+        scripts("determine_availability_matrix_MD_UA.py")
 
 
 # Optional input when having Ukraine (UA) or Moldova (MD) in the countries list
@@ -332,7 +332,7 @@ rule determine_availability_matrix:
     resources:
         mem_mb=config["atlite"].get("nprocesses", 4) * 5000,
     script:
-        "../scripts/determine_availability_matrix.py"
+        scripts("determine_availability_matrix.py")
 
 
 rule build_renewable_profiles:
@@ -367,7 +367,7 @@ rule build_renewable_profiles:
     wildcard_constraints:
         technology="(?!hydro).*",  # Any technology other than hydro
     script:
-        "../scripts/build_renewable_profiles.py"
+        scripts("build_renewable_profiles.py")
 
 
 rule build_monthly_prices:
@@ -387,7 +387,7 @@ rule build_monthly_prices:
     resources:
         mem_mb=5000,
     script:
-        "../scripts/build_monthly_prices.py"
+        scripts("build_monthly_prices.py")
 
 
 if COUNTRY_RUNOFF_DATASET["source"] == "build":
@@ -407,7 +407,7 @@ if COUNTRY_RUNOFF_DATASET["source"] == "build":
         benchmark:
             benchmarks("build_country_runoff")
         script:
-            "../scripts/build_country_runoff.py"
+            scripts("build_country_runoff.py")
 
 
 rule build_hydro_profile:
@@ -435,7 +435,7 @@ rule build_hydro_profile:
     resources:
         mem_mb=5000,
     script:
-        "../scripts/build_hydro_profile.py"
+        scripts("build_hydro_profile.py")
 
 
 rule build_line_rating:
@@ -459,7 +459,7 @@ rule build_line_rating:
     resources:
         mem_mb=config["atlite"].get("nprocesses", 4) * 1000,
     script:
-        "../scripts/build_line_rating.py"
+        scripts("build_line_rating.py")
 
 
 rule build_transmission_projects:
@@ -494,7 +494,7 @@ rule build_transmission_projects:
         mem_mb=4000,
     threads: 1
     script:
-        "../scripts/build_transmission_projects.py"
+        scripts("build_transmission_projects.py")
 
 
 rule add_transmission_projects_and_dlr:
@@ -532,7 +532,7 @@ rule add_transmission_projects_and_dlr:
     resources:
         mem_mb=4000,
     script:
-        "../scripts/add_transmission_projects_and_dlr.py"
+        scripts("add_transmission_projects_and_dlr.py")
 
 
 def input_class_regions(w):
@@ -564,7 +564,7 @@ rule build_electricity_demand_base:
     resources:
         mem_mb=5000,
     script:
-        "../scripts/build_electricity_demand_base.py"
+        scripts("build_electricity_demand_base.py")
 
 
 rule build_hac_features:
@@ -587,7 +587,7 @@ rule build_hac_features:
     resources:
         mem_mb=10000,
     script:
-        "../scripts/build_hac_features.py"
+        scripts("build_hac_features.py")
 
 
 rule process_cost_data:
@@ -608,7 +608,7 @@ rule process_cost_data:
     resources:
         mem_mb=4000,
     script:
-        "../scripts/process_cost_data.py"
+        scripts("process_cost_data.py")
 
 
 rule simplify_network:
@@ -643,7 +643,7 @@ rule simplify_network:
     resources:
         mem_mb=12000,
     script:
-        "../scripts/simplify_network.py"
+        scripts("simplify_network.py")
 
 
 # Optional input when using custom busmaps - Needs to be tailored to selected base_network
@@ -720,7 +720,7 @@ rule cluster_network:
     resources:
         mem_mb=10000,
     script:
-        "../scripts/cluster_network.py"
+        scripts("cluster_network.py")
 
 
 def input_profile_tech(w):
@@ -796,7 +796,7 @@ rule add_electricity:
     resources:
         mem_mb=10000,
     script:
-        "../scripts/add_electricity.py"
+        scripts("add_electricity.py")
 
 
 rule prepare_network:
@@ -832,7 +832,7 @@ rule prepare_network:
     resources:
         mem_mb=4000,
     script:
-        "../scripts/prepare_network.py"
+        scripts("prepare_network.py")
 
 
 if (
@@ -880,7 +880,7 @@ if (
         resources:
             mem_mb=4000,
         script:
-            "../scripts/clean_osm_data.py"
+            scripts("clean_osm_data.py")
 
     rule build_osm_network:
         message:
@@ -917,7 +917,7 @@ if (
         resources:
             mem_mb=4000,
         script:
-            "../scripts/build_osm_network.py"
+            scripts("build_osm_network.py")
 
 
 if config["electricity"]["base_network"] == "tyndp":
@@ -952,4 +952,4 @@ if config["electricity"]["base_network"] == "tyndp":
         resources:
             mem_mb=4000,
         script:
-            "../scripts/build_tyndp_network.py"
+            scripts("build_tyndp_network.py")