feat: add electricitymaps and entsoepy bidding zone data to data layer (#1991)

* feat: add electricitymaps and entsoepy bidding zone geospatial data to data layer

* config: autogenerate config.default.yaml and schema.json

* doc: add release note

* feat: use copy2 and update urls to be version specific

* feat: improve logger and handle raise timeout and url errors

* fix: move new versions.csv entries to the correct alphabetical position

* add to archive

---------

Co-authored-by: lkstrp <lkstrp@pm.me>

Author: daniel-rdt

config/config.default.yaml

@@ -1392,6 +1392,12 @@ data:
   jrc_ardeco:
     source: archive
     version: latest
+  bidding_zones_electricitymaps:
+    source: archive
+    version: latest
+  bidding_zones_entsoepy:
+    source: archive
+    version: latest
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#overpass_api
 overpass_api:

config/schema.json

@@ -1791,6 +1791,46 @@
               "type": "string"
             }
           }
+        },
+        "bidding_zones_electricitymaps": {
+          "description": "Configuration for a single data source.",
+          "properties": {
+            "source": {
+              "default": "archive",
+              "description": "Source of the data. 'archive' retrieves pre-built data, 'primary' retrieves from primary source.",
+              "enum": [
+                "archive",
+                "primary",
+                "build"
+              ],
+              "type": "string"
+            },
+            "version": {
+              "default": "latest",
+              "description": "Version of the data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source.",
+              "type": "string"
+            }
+          }
+        },
+        "bidding_zones_entsoepy": {
+          "description": "Configuration for a single data source.",
+          "properties": {
+            "source": {
+              "default": "archive",
+              "description": "Source of the data. 'archive' retrieves pre-built data, 'primary' retrieves from primary source.",
+              "enum": [
+                "archive",
+                "primary",
+                "build"
+              ],
+              "type": "string"
+            },
+            "version": {
+              "default": "latest",
+              "description": "Version of the data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source.",
+              "type": "string"
+            }
+          }
         }
       }
     },
@@ -12833,6 +12873,46 @@
               "type": "string"
             }
           }
+        },
+        "bidding_zones_electricitymaps": {
+          "description": "Configuration for a single data source.",
+          "properties": {
+            "source": {
+              "default": "archive",
+              "description": "Source of the data. 'archive' retrieves pre-built data, 'primary' retrieves from primary source.",
+              "enum": [
+                "archive",
+                "primary",
+                "build"
+              ],
+              "type": "string"
+            },
+            "version": {
+              "default": "latest",
+              "description": "Version of the data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source.",
+              "type": "string"
+            }
+          }
+        },
+        "bidding_zones_entsoepy": {
+          "description": "Configuration for a single data source.",
+          "properties": {
+            "source": {
+              "default": "archive",
+              "description": "Source of the data. 'archive' retrieves pre-built data, 'primary' retrieves from primary source.",
+              "enum": [
+                "archive",
+                "primary",
+                "build"
+              ],
+              "type": "string"
+            },
+            "version": {
+              "default": "latest",
+              "description": "Version of the data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source.",
+              "type": "string"
+            }
+          }
         }
       }
     },

data/versions.csv

@@ -7,18 +7,22 @@ bfs_gdp_and_population,2019-03-28,primary,latest supported,2025-12-02,,https://d
 bfs_gdp_and_population,2019-03-28,archive,latest supported,2025-12-02,,https://web.archive.org/web/20250818151254if_/https://dam-api.bfs.admin.ch/hub/api/dam/assets/7786557/master
 bfs_road_vehicle_stock,unknown,primary,latest supported,2025-12-02,This dataset is not versioned and is updated regularly. The link points to the latest version. More information on the latest update on this website: https://www.bfs.admin.ch/bfs/de/home/statistiken/kataloge-datenbanken.assetdetail.33827666.html,https://datawrapper.dwcdn.net/31f3521eddfa82ada1a436983c31caf5/1/dataset.csv
 bfs_road_vehicle_stock,2024-03-13,archive,latest supported,2025-12-02,,https://raw.githubusercontent.com/PyPSA/pypsa-eur/5b5d308bf70f15dd0b107d8a19c121093dcdd5bd/data/gr-e-11.03.02.01.01-cc.csv
+bidding_zones_electricitymaps,v1.238.0,primary,latest supported,2026-01-22,,https://raw.githubusercontent.com/electricitymaps/electricitymaps-contrib/refs/tags/v1.238.0/web/geo/world.geojson
+bidding_zones_electricitymaps,v1.238.0,archive,latest supported,2026-01-22,,https://data.pypsa.org/workflows/eur/bidding_zones_electricitymaps/v1.238.0/world.geojson
+bidding_zones_entsoepy,v0.6.18,primary,latest supported,2026-01-22,,https://raw.githubusercontent.com/EnergieID/entsoe-py/refs/tags/V0.6.18/entsoe/geo/geojson
+bidding_zones_entsoepy,v0.6.18,archive,latest supported,2026-01-22,,https://data.pypsa.org/workflows/eur/bidding_zones_entsoepy/v0.6.18
 co2stop,26-august-2020,primary,latest supported,2025-12-02,,https://setis.ec.europa.eu/document/download/786a884f-0b33-4789-b744-28004b16bd1a_en?filename=co2jrc_openformats.zip
 co2stop,26-august-2020,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/co2stop/26-august-2020/co2jrc_openformats.zip
 copernicus_land_cover,v3.0.1,primary,latest supported,2025-12-02,"The primary is already from Zenodo, documentation in https://zenodo.org/records/4723921",https://zenodo.org/records/3939050/files/PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif
 copernicus_land_cover,v3.0.1,archive,latest supported,2026-01-20,"The primary is already from Zenodo, documentation in https://zenodo.org/records/4723921",https://data.pypsa.org/workflows/eur/copernicus_land_cover/v3.0.1
 copernicus_land_cover,v2.0.2,archive,deprecated supported,2026-01-13,,https://data.pypsa.org/workflows/eur/copernicus_land_cover/v2.0.2/PROBAV_LC100_global_v3.0.1_2015-base_Discrete-Classification-map_EPSG-4326.tif
 corine,v18_5,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/corine/v18_5/corine.zip
 corine,unknown,primary,latest supported,2025-12-02,Need to register with CLMS API and create an access token. The download URL is dynamic,
-costs,v0.13.4,primary,latest supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/v0.13.4/outputs
+costs,v0.13.4,primary,latest supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/refs/tags/v0.13.4/outputs
 costs,v0.13.4,archive,supported,2026-01-21,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://data.pypsa.org/workflows/eur/costs/v0.13.4
-costs,v0.13.3,primary,supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/v0.13.3/outputs
+costs,v0.13.3,primary,supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/refs/tags/v0.13.3/outputs
 costs,v0.13.3,archive,supported,2026-01-20,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://data.pypsa.org/workflows/eur/costs/v0.13.3
-costs,v0.13.2,primary,supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/v0.13.2/outputs
+costs,v0.13.2,primary,supported,2025-12-02,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://raw.githubusercontent.com/PyPSA/technology-data/refs/tags/v0.13.2/outputs
 costs,v0.13.2,archive,supported,2026-01-20,"Part of the `technologydata` repository and versioned on GitHub, i.e. no dedicated 'archive' entry.",https://data.pypsa.org/workflows/eur/costs/v0.13.2
 country_hdd,unknown,build,latest supported,2025-12-02,Latest dataset built using ERA5 runoff data. This takes a very long time to build.,
 country_hdd,2025-08-13,primary,latest supported,2025-12-02,,https://zenodo.org/records/16849356/files/era5-HDD-per-country.csv

doc/data_inventory.csv

@@ -41,4 +41,6 @@
 "mobility_profiles","German Vehicle Activity Profiles","Vehicle activity profiles for different vehicle types and road types in Germany, based on monitoring data from the Federal Highway Research Institute (BASt). These profiles provide insights into travel behavior and patterns, which can be used for transport modeling and analysis.","Federal Highway Research Institute (BASt)","https://www.bast.de/DE/Themen/Digitales/HF_1/Massnahmen/verkehrszaehlung/Stundenwerte.html?nn=414410","CC-BY-4.0"
 "dh_areas","","Shapes of district heating areas","ISI Fraunhofer-Institut für System- und Innovationsforschung","https://fordatis.fraunhofer.de/handle/fordatis/341.5","CC-BY-4.0"
 "geothermal_heat_utilisation_potentials","","Potentials for Geothermal heat utilisation","ISI Fraunhofer-Institut für System- und Innovationsforschung","https://fordatis.fraunhofer.de/handle/fordatis/341.5","CC-BY-4.0"
-"jrc_ardeco","Annual Regional Database of the European Commission's Directorate General for Regional and Urban Policy","The database contains a set of long time-series variables and indicators for EU regions, as well as for regions in some EFTA and candidate countries, at various statistical scales (NUTS1, NUTS2, NUTS3, metro regions).","European Commission","https://territorial.ec.europa.eu/ardeco","similar to CC-BY"
+"jrc_ardeco","Annual Regional Database of the European Commission's Directorate General for Regional and Urban Policy","The database contains a set of long time-series variables and indicators for EU regions, as well as for regions in some EFTA and candidate countries, at various statistical scales (NUTS1, NUTS2, NUTS3, metro regions).","European Commission","https://territorial.ec.europa.eu/ardeco","similar to CC-BY"
+"bidding_zones_electricitymaps","Electricity Maps Bidding Zones","Geospatial data defining bidding zones for electricity markets in Europe","Electricity Maps","https://github.com/electricitymaps/electricitymaps-contrib","AGPL-3.0"
+"bidding_zones_entsoepy","ENTSOE-PY Bidding Zones","Geospatial data defining bidding zones for electricity markets in Europe","EnergieID","https://github.com/EnergieID/entsoe-py","MIT"

doc/release_notes.rst

@@ -9,6 +9,8 @@ Release Notes
 Upcoming Release
 ================
 
+* Added missing bidding zone data sources to data layer (https://github.com/PyPSA/pypsa-eur/pull/1991).
+
 * Fix virtual bus naming when building the transmission network from raw OSM data to use persistent names (https://github.com/PyPSA/pypsa-eur/pull/1956).
 
 * Fix column selection when preparing OSM pre-built releases (https://github.com/PyPSA/pypsa-eur/pull/1956).

rules/retrieve.smk

@@ -319,19 +319,62 @@ if (EU_NUTS2021_DATASET := dataset_version("eu_nuts2021"))["source"] in [
             unpack_archive(output["zip_file"], Path(output.shapes_level_3).parent)
 
 
-rule retrieve_bidding_zones:
-    message:
-        "Retrieving bidding zones data from ENTSO-E and Electricity Maps"
-    output:
-        file_entsoepy="data/busshapes/bidding_zones_entsoepy.geojson",
-        file_electricitymaps="data/busshapes/bidding_zones_electricitymaps.geojson",
-    log:
-        "logs/retrieve_bidding_zones.log",
-    resources:
-        mem_mb=1000,
-    retries: 2
-    script:
-        "../scripts/retrieve_bidding_zones.py"
+if (
+    BIDDING_ZONES_ELECTRICITYMAPS_DATASET := dataset_version(
+        "bidding_zones_electricitymaps"
+    )
+)["source"] in ["primary"]:
+
+    rule retrieve_bidding_zones_electricitymaps:
+        input:
+            geojson=storage(BIDDING_ZONES_ELECTRICITYMAPS_DATASET["url"]),
+        output:
+            geojson=f"{BIDDING_ZONES_ELECTRICITYMAPS_DATASET["folder"]}/bidding_zones_electricitymaps.geojson",
+        log:
+            "logs/retrieve_bidding_zones_electricitymaps.log",
+        resources:
+            mem_mb=1000,
+        retries: 2
+        run:
+            copy2(input["geojson"], output["geojson"])
+
+
+if (BIDDING_ZONES_ENTSOEPY_DATASET := dataset_version("bidding_zones_entsoepy"))[
+    "source"
+] in ["primary"]:
+
+    rule retrieve_bidding_zones_entsoepy:
+        output:
+            geojson=f"{BIDDING_ZONES_ENTSOEPY_DATASET['folder']}/bidding_zones_entsoepy.geojson",
+        log:
+            "logs/retrieve_bidding_zones_entsoepy.log",
+        resources:
+            mem_mb=1000,
+        retries: 2
+        run:
+            import entsoe
+            import geopandas as gpd
+            from urllib.error import HTTPError, URLError
+
+            logger.info("Downloading entsoe-py zones...")
+            gdfs: list[gpd.GeoDataFrame] = []
+            url = f"{BIDDING_ZONES_ENTSOEPY_DATASET['url']}"
+            for area in entsoe.Area:
+                name = area.name
+                try:
+                    file_url = f"{url}/{name}.geojson"
+                    gdfs.append(gpd.read_file(file_url))
+                except HTTPError as e:
+                    logger.debug(f"Area file not available for {name}: {e}")
+                    continue
+                except (URLError, TimeoutError) as e:
+                    raise Exception(f"Network error retrieving {name}: {e}")
+            shapes = pd.concat(gdfs, ignore_index=True)  # type: ignore
+
+            logger.info("Downloading entsoe-py zones... Done")
+
+            shapes.to_file(output.geojson)
+
 
 
 if (CUTOUT_DATASET := dataset_version("cutout"))["source"] in [

scripts/lib/validation/config/data.py

@@ -219,3 +219,11 @@ class DataConfig(BaseModel):
         default_factory=_DataSourceConfig,
         description="JRC ARDECO data source configuration.",
     )
+    bidding_zones_electricitymaps: _DataSourceConfig = Field(
+        default_factory=_DataSourceConfig,
+        description="Electricitymaps bidding zones data source configuration.",
+    )
+    bidding_zones_entsoepy: _DataSourceConfig = Field(
+        default_factory=_DataSourceConfig,
+        description="Entsoepy bidding zones data source configuration.",
+    )