Update and clean up Snakefiles and integration_test.

Author: bobbyxng

tests/integration/Snakefile

@@ -2,37 +2,20 @@
 configfile: workflow.source_path("./test_config.yaml")
 
 
-# Import the module and configure it.
-# `snakefile:` specifies the module. It can use file paths and special github(...) / gitlab(...) markers
-# `config`: specifies the module configuration.
-# `pathvars:` helps you re-wire where the module places files.
 module grid_builder:
     pathvars:
-        # Redirect OSM outputs
-        osm_out="resources/module/resources/automatic/osm/out",
-        # Redirect intermediate files that are internal to the module
-        logs="resources/module/logs",
-        resources="resources/module/resources",
-        results="resources/module/results",
+        resources="resources/grid-builder",
+        logs="logs/grid-builder",
     snakefile:
         "../../workflow/Snakefile"
     config:
         config["grid_builder"]
 
 
-# rename all module rules with a prefix, to avoid naming conflicts.
 use rule * from grid_builder as grid_builder_*
 
 
-# Request OSM retrieval outputs from the module
 rule all:
     default_target: True
     input:
-        expand(
-            "resources/module/resources/automatic/osm/out/{country}_{feature}.{ext}",
-            country="belgium",
-            feature=["substation", "line"],
-            ext=["csv", "geojson"],
-        ),
-    message:
-        "Retrieve OSM grid data by country and feature."
+        rules.grid_builder_retrieve_osm_all.input,

tests/integration/test_config.yaml

@@ -1,6 +1,6 @@
 grid_builder:
   countries:
-    - belgium
+    - benin
 
   retrieve:
     source: geofabrik
@@ -12,4 +12,4 @@ grid_builder:
     mp: true
     stream_backend: true
     cache_primary: false
-    target_date: null
+    target_date:

tests/integration_test.py

@@ -40,14 +40,6 @@ def test_standard_file_existance(module_path, file):
     assert Path(module_path / file).exists()
 
 
-def test_snakemake_all_failure(module_path):
-    """The snakemake 'all' rule should return an error by default."""
-    process = subprocess.run(
-        "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True
-    )
-    assert "INVALID (missing locally)" in str(process.stderr)
-
-
 def test_snakemake_integration_testing(module_path):
     """Run a light-weight test simulating someone using this module."""
     assert subprocess.run(

workflow/Snakefile

@@ -12,7 +12,7 @@ min_version("9.19")
 # Define pathvars to expose OSM retrieval outputs for downstream use.
 pathvars:
     # OSM retrieval outputs by country and feature
-    osm_out="<resources>/automatic/osm/out",
+    osm_out="<resources>/osm/out",
 
 
 # Default configuration file generated from pydantic schema.

workflow/rules/retrieve.smk

@@ -6,12 +6,12 @@
 rule retrieve_osm:
     output:
         csv=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.csv",
+            "<resources>/osm/out/{country}_{feature}.csv",
             country="{country}",
             feature=config["retrieve"]["features"],
         ),
         geojson=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.geojson",
+            "<resources>/osm/out/{country}_{feature}.geojson",
             country="{country}",
             feature=config["retrieve"]["features"],
         ),
@@ -37,12 +37,12 @@ rule retrieve_osm:
 rule retrieve_osm_all:
     input:
         csv=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.csv",
+            "<resources>/osm/out/{country}_{feature}.csv",
             country=config["countries"],
             feature=config["retrieve"]["features"],
         ),
         geojson=expand(
-            "<resources>/automatic/osm/out/{country}_{feature}.geojson",
+            "<resources>/osm/out/{country}_{feature}.geojson",
             country=config["countries"],
             feature=config["retrieve"]["features"],
         ),