Squashed commit of:

adjust config to upstream changes

adjust to upstream

up pixi.lock

hotfix pydantic config

fix: don't mutate config

adjust to removed meoh config

impute Datein or drop chps with missing DateIn

fix import

fix typo

skip writing constraints if there are no corresponding assets in that year

fix: bypass cached http plugin for egon data retrieval

update lock file

fix: pin pypsa<1.1

adjust existing assets to upstream config

fix: robustness against empty arrays of links

overwrite values from add_existing_baseyear during offshore nep force

add a delay for offshore projects in 2020-2025 to correspond better to real capacities

overwrite existing offwind more aggresively

Author: lindnemi

Snakefile

@@ -422,20 +422,19 @@ if (ARIADNE_DATABASE := dataset_version("ariadne_database"))["source"] in ["arch
 
 if (ARIADNE_TEMPLATE := dataset_version("ariadne_template"))["source"] in [
     "primary",
+    "archive",
 ]:
 
     rule retrieve_ariadne_template:
         input:
-            storage(
-                "https://github.com/iiasa/ariadne-intern-workflow/raw/main/attachments/2025-01-27_template_Ariadne.xlsx",
-            ),
+            storage(ARIADNE_TEMPLATE["url"]),
         output:
             "data/template_ariadne_database.xlsx",
         run:
             move(input[0], output[0])
 
 
-if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary"]:
+if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary", "archive"]:
 
     rule retrieve_open_mastr:
         input:
@@ -449,22 +448,35 @@ if (OPEN_MASTR := dataset_version("open_mastr"))["source"] in ["primary"]:
             unpack_archive(input[0], params[0])
 
 
-if (EGON := dataset_version("egon"))["source"] in ["primary"]:
+if (EGON := dataset_version("egon"))["source"] in ["build"]:
+
+    rule retrieve_egon_data:
+        params:
+            url=EGON["url"],
+            folder=EGON["folder"],
+        output:
+            spatial=f"{EGON['folder']}/demandregio_spatial_2018.json",
+            mapping=f"{EGON['folder']}/mapping_technologies.json",
+        shell:
+            """
+            mkdir -p {params.folder}
+            curl -o {output.spatial} "{params.url}?id_spatial=5&year=2018"
+            curl -o {output.mapping} "{params.url}_description?id_spatial=5"
+            """
+
+
+if (EGON := dataset_version("egon"))["source"] in ["archive"]:
 
     rule retrieve_egon_data:
         input:
-            spatial=storage(
-                f"{EGON['url']}?id_spatial=5&year=2018",
-            ),
-            mapping=storage(
-                f"{EGON['url']}_description?id_spatial=5",
-            ),
+            spatial=storage(f"{EGON['url']}/demandregio_spatial_2018.json"),
+            mapping=storage(f"{EGON['url']}/mapping_technologies.json"),
         output:
-            spatial="data/egon/demandregio_spatial_2018.json",
-            mapping="data/egon/mapping_technologies.json",
+            spatial=f"{EGON['folder']}/demandregio_spatial_2018.json",
+            mapping=f"{EGON['folder']}/mapping_technologies.json",
         run:
-            move(input.spatial, output.spatial)
-            move(input.mapping, output.mapping)
+            copy2(input["spatial"], output["spatial"])
+            copy2(input["mapping"], output["mapping"])
 
 
 rule build_exogenous_mobility_data:
@@ -494,12 +506,12 @@ rule build_exogenous_mobility_data:
 
 rule build_egon_data:
     input:
-        demandregio_spatial="data/egon/demandregio_spatial_2018.json",
+        demandregio_spatial=f"{EGON['folder']}/demandregio_spatial_2018.json",
         mapping_38_to_4=storage(
             "https://ffeopendatastorage.blob.core.windows.net/opendata/mapping_from_4_to_38.json",
             keep_local=True,
         ),
-        mapping_technologies="data/egon/mapping_technologies.json",
+        mapping_technologies=f"{EGON['folder']}/mapping_technologies.json",
         nuts3=resources("nuts3_shapes.geojson"),
     output:
         heating_technologies_nuts3=resources("heating_technologies_nuts3.geojson"),
@@ -656,7 +668,6 @@ rule modify_prenetwork:
         ),
         shipping_oil_efficiency=config_provider("sector", "shipping_oil_efficiency"),
         shipping_methanol_share=config_provider("sector", "shipping_methanol_share"),
-        mwh_meoh_per_tco2=config_provider("sector", "MWh_MeOH_per_tCO2"),
         scale_capacity=config_provider("scale_capacity"),
         bev_charge_rate=config_provider("sector", "bev_charge_rate"),
         bev_energy=config_provider("sector", "bev_energy"),
@@ -927,7 +938,7 @@ rule plot_ariadne_variables:
 
 rule ariadne_all:
     input:
-        expand(RESULTS + "graphs/costs.svg", run=config_provider("run", "name")),
+        expand(RESULTS + "graphs/costs.pdf", run=config_provider("run", "name")),
         # expand(
         #     RESULTS + "ariadne/capacity_detailed.png",
         #     run=config_provider("run", "name"),

config/config.de.yaml

@@ -423,6 +423,7 @@ solving:
   runtime: 12h
   mem_mb: 70000 #30000 is OK for 22 nodes, 365H; 140000 for 22 nodes 3H; 400000 for 44 nodes 3H
   options:
+    custom_extra_functionality: "../scripts/pypsa-de/additional_functionality.py"
     assign_all_duals: true
     load_shedding: false
     skip_iterations: true # settings for post-discretization: false
@@ -471,7 +472,7 @@ solving:
         offwind:
           DE:
             2020: 7.8
-            2025: 12
+            2025: 9.7  # https://www.offshore-stiftung.de/de/status-quo-offshore-windenergie.php
             2030: 22   # optimistic upper ceiling to "Mittelfristprognose zur deutschlandweiten Stromerzeugung"
             2035: 50   # Planned projects until 2035 (offshore_connection_points.csv) -1.3 GW for potential delays
             2040: 65   # Planned projects until 2040 -1.5 GW for potential retirments
@@ -658,11 +659,9 @@ plotting:
 
 electricity:
   renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, hydro] # removed offwind-float
-  powerplants_filter: (DateOut >= 2024 or DateOut != DateOut) and not (Country == "DE" and Set == "CHP")
+  powerplants_filter: (DateOut > 2025 or DateOut != DateOut) and not (Country == "DE" and Set == "CHP") and (DateIn < 2026 or DateIn != DateIn)
   custom_powerplants: true
   custom_file: resources/german_chp.csv
-  estimate_renewable_capacities:
-    year: 2024
   H2_plants:
     enable: EU # allowed values: false, DE, EU
     start: 2030 # should be < force
@@ -718,11 +717,11 @@ data:
     source: archive
     version: latest
   ariadne_template:
-    source: primary
+    source: archive
     version: latest
   open_mastr:
-    source: primary
+    source: archive
     version: latest
   egon:
-    source: primary
+    source: archive
     version: latest

config/test/config.dach.yaml

@@ -44,7 +44,7 @@ renewable:
     cutout: dach-03-2013-sarah3-era5
 
 clustering:
-  focus_weights: []
+  focus_weights: false
   temporal:
     resolution_sector: 3H
 

data/versions.csv

@@ -1,11 +1,10 @@
 dataset,version,source,tags,added,note,url
-ariadne_database,v1.0,primary,latest supported,2026-01-15,Public facing database of the second Ariadne Szenarienbericht,https://ariadne2.apps.ece.iiasa.ac.at/explorer
-ariadne_database,v1.0,archive,latest supported,2026-01-15,,https://zenodo.org/records/15174592/files/250505_Ariadne2_Data_v1.0.xlsx
-ariadne_template,2025-11-20,primary,latest supported,2026-01-15,,https://github.com/iiasa/ariadne-intern-workflow/blob/main/attachments/2025-11-20_template_Ariadne.xlsx
-egon,2018-2020?,primary,latest supported,2026-01-15,root url is extended in the retrieve rule,https://api.opendata.ffe.de/demandregio/demandregio_spatial
-open_mastr,2023-08-08,primary,latest supported,2026-01-15,,https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip
 aquifer_data,v1.2,primary,latest supported,2025-12-02,,https://download.bgr.de/bgr/grundwasser/IHME1500/v12/shp/IHME1500_v12.zip
 aquifer_data,v1.2,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/aquifer_data/v1.2/IHME1500_v12.zip
+ariadne_database,v1.0,primary,latest supported,2026-01-15,Public facing database of the second Ariadne Szenarienbericht,https://zenodo.org/records/15174592/files/250505_Ariadne2_Data_v1.0.xlsx
+ariadne_database,v1.0,archive,latest supported,2026-02-23,Public facing database of the second Ariadne Szenarienbericht,https://data.pypsa.org/workflows/de/ariadne_database/v1.0/250505_Ariadne2_Data_v1.0.xlsx
+ariadne_template,2025-11-20,primary,latest supported,2026-01-15,,https://github.com/iiasa/ariadne-intern-workflow/raw/7caa948ad6aa71cf8d17e69a64dd29f01306a9ff/attachments/2025-11-20_template_Ariadne.xlsx
+ariadne_template,2025-11-20,archive,latest supported,2026-02-23,,https://data.pypsa.org/workflows/de/ariadne_template/2025-11-20/2025-11-20_template_Ariadne.xlsx
 attributed_ports,2020-07-10,primary,latest supported,2025-12-02,,https://datacatalogfiles.worldbank.org/ddh-published/0038118/1/DR0046414/attributed_ports.geojson
 attributed_ports,2020-07-10,archive,latest supported,2026-01-13,Moved from github repo `data/` folder,https://data.pypsa.org/workflows/eur/attributed_ports/2020-07-10/attributed_ports.json
 bfs_gdp_and_population,2019-03-28,primary,latest supported,2025-12-02,,https://dam-api.bfs.admin.ch/hub/api/dam/assets/7786557/master
@@ -45,6 +44,8 @@ dh_areas,341.5,primary,latest supported,2025-12-02,,https://fordatis.fraunhofer.
 dh_areas,341.5,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/dh_areas/341.5/dh_areas.gpkg
 eez,v12_20231025,primary,latest supported,2025-12-02,API request used,https://www.marineregions.org/download_file.php
 eez,v12_20231025,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/eez/v12_20231025/World_EEZ_v12_20231025_LR.zip
+egon,2019-10-02,archive,latest supported,2026-02-23,,https://data.pypsa.org/workflows/de/egon/2019-10-02
+egon,2019-10-02,build,latest supported,2026-01-15,root url is extended in the retrieve rule,https://api.opendata.ffe.de/demandregio/demandregio_spatial
 emobility,28-08-2016,primary,latest supported,2025-12-02,,https://zenodo.org/records/16899168/files/emobility.zip
 emobility,28-08-2016,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/emobility/28-08-2016/emobility.zip
 enspreso_biomass,unknown,primary,latest supported,2025-12-02,"2019 version was changed in 2023 with negligible differences, but since the version is not frozen we label it as 'unknown'.",https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx
@@ -113,6 +114,8 @@ nuts3_population,13-03-2025,primary,latest supported,2025-12-02,,https://ec.euro
 nuts3_population,13-03-2025,archive,latest supported,2026-01-13,earlier part of zenodo bundle,https://data.pypsa.org/workflows/eur/nuts3_population/13-03-2025/nama_10r_3popgdp.tsv.gz
 ons_lad,may-2024,primary,latest supported,2026-02-02,API request used,https://services1.arcgis.com/ESMARspQHYMw9BZ9/arcgis/rest/services/Local_Authority_Districts_May_2024_Boundaries__UK_BSC/FeatureServer/0/query
 ons_lad,may-2024,archive,latest supported,2026-02-17,API request used,https://data.pypsa.org/workflows/eur/ons_lad/may-2024/ons_lad.geojson
+open_mastr,2023-08-08,primary,latest supported,2026-01-15,,https://zenodo.org/records/8225106/files/bnetza_open_mastr_2023-08-08_B.zip
+open_mastr,2023-08-08,archive,latest supported,2026-02-23,,https://data.pypsa.org/workflows/de/open_mastr/2023-08-08/bnetza_open_mastr_2023-08-08_B.zip
 opsd_electricity_demand,unknown,build,latest supported,2026-02-02,multiple versions concatenated,https://data.open-power-system-data.org/time_series/
 opsd_electricity_demand,2026-02-02,archive,latest supported,2026-02-03,,https://data.pypsa.org/workflows/eur/opsd_electricity_demand/2026-02-02/electricity_demand_opsd_raw.csv
 osm,unknown,build,latest supported,2025-12-02,Latest dataset built using OSM data. No single URL available to access the data. See the related scripts and documentation for details.,
@@ -164,4 +167,4 @@ wdpa_marine,2025-07,archive,latest supported,2025-12-02,"We are legally not allo
 worldbank_commodity_prices,jan-2026,primary,latest supported,2026-02-03,,https://thedocs.worldbank.org/en/doc/18675f1d1639c7a34d463f59263ba0a2-0050012025/related/CMO-Historical-Data-Monthly.xlsx
 worldbank_commodity_prices,jan-2026,archive,latest supported,2026-02-12,,https://data.pypsa.org/workflows/eur/worldbank_commodity_prices/jan-2026/CMO-Historical-Data-Monthly.xlsx
 worldbank_urban_population,unknown,primary,latest might-work,2025-12-02,"This is the original World Bank API link, which is sometimes updated; it is not guaranteed to work with the current codebase and data changes without notice.",https://api.worldbank.org/v2/en/indicator/SP.URB.TOTL.IN.ZS?downloadformat=csv
-worldbank_urban_population,2025-08-14,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/worldbank_urban_population/2025-08-14/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_22447.zip
+worldbank_urban_population,2025-08-14,archive,latest supported,2026-01-13,,https://data.pypsa.org/workflows/eur/worldbank_urban_population/2025-08-14/API_SP.URB.TOTL.IN.ZS_DS2_en_csv_v2_22447.zip