diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 18ba1af..cd7249b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -98,6 +98,8 @@ jobs: TALSPERREN_DATAWRAPPER_TOKEN: ${{ secrets.NASA_WALDBRANDDATEN_RHODOS_DATAWRAPPER_TOKEN }} DB_CLIENT_ID: ${{ secrets.DB_CLIENT_ID }} DB_API_KEY: ${{ secrets.DB_API_KEY }} + # Datawrapper key for Jan Eggers' account + DW_API_KEY_JE: ${{ secrets.DW_API_KEY_JE }} - name: Log artifact sizes run: | diff --git a/.vscode/extensions.json b/.vscode/extensions.json index c1ea73e..800730f 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -2,6 +2,7 @@ "recommendations": [ "charliermarsh.ruff", "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", + "continue.continue" ] } diff --git a/README.md b/README.md index 4751ca6..a9d8a28 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Find the scraper you created in the `ddj_cloud/scrapers` folder and open the `.p You can run the following command to test your scraper: uv run manage test - + where `` is the Python module name of your scraper. If a local `.env` file exists in the repository root, `manage test` will load it automatically before importing the scraper. diff --git a/ddj_cloud/scrapers/klimadashboard/.claude/settings.json b/ddj_cloud/scrapers/klimadashboard/.claude/settings.json new file mode 100644 index 0000000..661a262 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/.claude/settings.json @@ -0,0 +1,13 @@ +{ + "permissions": { + "allow": [ + "WebFetch(domain:github.com)", + "WebFetch(domain:raw.githubusercontent.com)", + "Read(//Users/janeggers/Code/wdr-ddj-cloud/ddj_cloud/scrapers/talsperren/**)", + "WebFetch(domain:open-mastr.readthedocs.io)", + "WebFetch(domain:api.github.com)", + "Read(//Users/janeggers/miniconda3/lib/python3.12/site-packages/open_mastr/**)", + "Read(//Users/janeggers/Code/wdr-ddj-cloud/**)" + ] + } +} diff --git a/ddj_cloud/scrapers/klimadashboard/.copier-answers.yml b/ddj_cloud/scrapers/klimadashboard/.copier-answers.yml new file mode 100644 index 0000000..2c726d8 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/.copier-answers.yml @@ -0,0 +1,14 @@ +# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY +_src_path: /Users/janeggers/Code/wdr-ddj-cloud/scraper_template +contact_email: jan.eggers@fm.wdr.de +contact_name: Jan Eggers +description: 'Automation für Quarks.de: Ausbau von Wind- und Solarenergie, Energiemix + in D und mehr + + ' +display_name: klimadashboard +ephemeral_storage: '512' +interval: daily +memory_size: '1024' +preset: pandas + diff --git a/ddj_cloud/scrapers/klimadashboard/.gitignore b/ddj_cloud/scrapers/klimadashboard/.gitignore new file mode 100644 index 0000000..ce51518 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/.gitignore @@ -0,0 +1,2 @@ +# MaStR databases +*.db diff --git a/ddj_cloud/scrapers/klimadashboard/CLAUDE.md b/ddj_cloud/scrapers/klimadashboard/CLAUDE.md new file mode 100644 index 0000000..6471349 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/CLAUDE.md @@ -0,0 +1,52 @@ +# Technology stack + +- Python 3.11 +- uv +- Datawrapper (Charts) +- SQLite Database (MaStR-Daten) +- Fraunhofer Energy Charts API (Energiemix) +- MaStR SOAP API (Windkraft-Ausbau) +- Sentry (Monitoring) + +## Step 1: POC Datawrapper -- DONE + +- Look at src/energiemix.py which is a crude sample of a gather-process-store-publish pipeline +- Look for errors and improve the code + +### Findings & Fixes (2026-03-30) + +7 bugs fixed in `src/energiemix.py`: +1. `MIX_NOTES` was defined twice, shadowing `POWER_NOTES` -> renamed second to `POWER_NOTES` +2. `fetch_public_power()` called wrong API endpoint (`SHARE_FORECAST` instead of `PUBLIC_POWER`) +3. `upload_to_datawrapper()` used undefined `DATAWRAPPER_CHART_ID` -> changed to `dw_id` param +4. Column selection used tuple syntax instead of list (`df["a", "b"]` -> `df[["a", "b"]]`) +5. `POWER_NOTES` was undefined because of bug #1 +6. Raw DataFrame passed to `upload_to_datawrapper` instead of CSV -> added `build_csv_from_index()` +7. Returned CSV string but caller expected DataFrame -> now returns `df_combined` + +## Step 2: PHP to Python -- DONE + +- Look at the msr_php subfolder containing PHP scripts to scrape and process wind data +- Construct a Python version of it "msr_wind.py", analog to the src/energiemix.py +- Document in README_msr.md, noting all secrets and keys needed +- Suggest msr_solar.py for solar energy + +### Findings (2026-03-30) + +Created `src/msr_wind.py` porting `msr_php/wka_daily.php` + `msr_php/wka_to_data.php`: +- Uses SQLite instead of MySQL, requests instead of PHP SoapClient, pandas instead of per-row SQL +- `fetch_recent_units()`: fetches new/updated wind units from MaStR API +- `process_daily_data()`: calculates daily capacity (installed, planned, required for 2030 targets) +- Documented all secrets in `README_msr.md` +- Solar suggestion included in README_msr.md (energietraeger: "Solare Strahlungsenergie", 215 GW target) + +## Step 3: Add monitoring -- DONE + +- Look at the ../../utils to understand sentry +- Add useful sentry functions + +### Findings (2026-03-30) + +Added `sentry_sdk.capture_exception(e)` to all API calls in both files: +- `energiemix.py`: all 4 Fraunhofer API fetch functions +- `msr_wind.py`: SOAP API call + per-unit error handling (individual failures don't crash the run) \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/README.md b/ddj_cloud/scrapers/klimadashboard/README.md new file mode 100644 index 0000000..487ccd2 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/README.md @@ -0,0 +1,112 @@ +# klimadashboard + +**Contact:** Jan Eggers (jan.eggers@fm.wdr.de) + +Automation für Quarks.de: Ausbau von Wind- und Solarenergie, Energiemix in D und mehr + +## Architektur + +``` +klimadashboard.py (Orchestrator) + │ + ├── msr_scraper.py → alle Energiearten aus MaStR (isoliertes venv via uv run) + ├── msr_wind_processor.py → Wind-Tagesdaten berechnen + ├── msr_solar_processor.py → Solar-Tagesdaten berechnen + ├── msr_dw_display.py → Datawrapper-Charts aktualisieren + ├── S3: upload mastr.db + CSVs + └── energiemix.py → Fraunhofer-Daten + DW-Charts +``` + + +## MaStR-Scraper; Auswertung Wind- und Solarenergie + +Ausbaustand Wind- und Solarenenergie: Wie geht es voran? Was muss passieren, um die Ziele des EEG zu erreichen? + +Ursprünglich ein Python-Port der PHP-Skripte `msr_php/wka_daily.php` und `msr_php/wka_to_data.php`, jetzt basierend auf der [open-mastr](https://github.com/OpenEnergyPlatform/open-mastr)-Bibliothek des [Rainer-Lemoine-Instituts](https://wam.rl-institut.de/#showcase). Die Maintainer dort sind Jonathan Amme und Ludwig Hülk - die das mehr oder weniger nebenbei entwickeln und für Props und Kooperationen offen sind. + + +### 1. Scraper (`src/msr_scraper.py`) + +Lädt alle Energiearten (Wind, Solar, Biomasse, Wasser, Kernkraft, Verbrennung, Geothermie/Grubengas, Speicher) +über den open-mastr Bulk-Download und speichert sie in `mastr.db`. + +**Kein API-Key nötig** -- nutzt die öffentlichen Bulk-Daten des MaStR. + +**Isoliertes venv:** Der Scraper nutzt PEP 723 inline script metadata und wird via `uv run` +in einem eigenen virtuellen Environment ausgeführt (open-mastr benötigt pandas>=2.2, +das Hauptprojekt nutzt pandas~=1.5). + +**Caching:** Wenn `mastr.db` bereits Daten von heute enthält (`DatumDownload`), wird der Download übersprungen. + +### 2. Wind-Prozessor (`src/msr_wind_processor.py`) + +Berechnet tägliche Ausbaudaten (2010-2030) für Onshore und Offshore Wind: +- Kumulierte installierte Leistung (GW) +- Täglicher Zubau/Abbau (MW) +- Geplante zukünftige Installationen +- Nötiger täglicher Ausbau für die Klimaschutzziele 2030 +- Monatliche und jährliche Zusammenfassungen + +**Klimaziele 2030:** +- Onshore: 115 GW (Wind-an-Land-Gesetz, seit 01.02.2023) +- Offshore: 30 GW (Wind-auf-See-Gesetz, seit 01.01.2023) + +### 3. Solar-Prozessor (`src/msr_solar_processor.py`) + +Berechnet tägliche Ausbaudaten (2010-2030) für Solarenergie: +- Kumulierte installierte Leistung (GW) +- Täglicher Zubau/Abbau (MW) +- Geplante zukünftige Installationen +- Nötiger täglicher Ausbau für das Klimaziel 2030 +- Monatliche und jährliche Zusammenfassungen + +**Klimaziel 2030:** 215 GW (EEG 2023) + +### 4. Datawrapper-Display (`src/msr_dw_display.py`) + +Lädt aufbereitete Daten in Datawrapper-Charts hoch: +- **Wind-Ausbau** (`EgOti`): Gesamtleistung Onshore/Offshore +- **Solar-Ausbau** (`1rxLQ`): Gesamtleistung Solar +- **Wind-Zubau** (`7yMTK`): Zubau pro Monat/Jahr +- **Solar-Zubau** (`kPzGf`): Zubau pro Monat/Jahr + +Umschaltbar zwischen monatlicher und jährlicher Aggregation via `YEARLY_AGGREGATES`. + +## Benötigte Secrets / Umgebungsvariablen + +| Variable | Beschreibung | Wo? | +|----------|-------------|-----| +| `DATAWRAPPER_API_KEY` | API-Token für Datawrapper-Charts | [Datawrapper Account Settings](https://app.datawrapper.de/account/api-tokens), in .env des Projekts | + +Der Upload ins S3-Bucket erfolgt über eine Bibliotheksfunktion des Projekts; keine Extra-Keys nötig. + +## Datenbank + +Die SQLite-Datenbank `mastr.db` liegt in `local_storage/klimadashboard/` und wird nach Verarbeitung auf S3 hochgeladen. + +**Tabellen aus MaStR** (open-mastr-Schema): +- `wind_extended`, `solar_extended`, `biomass_extended`, `hydro_extended`, + `combustion_extended`, `nuclear_extended`, `gsgk_extended`, `storage_extended` + +**Berechnete Tabellen:** +- `ee_wind_taeglich`: Tägliche Wind-Ausbaudaten pro Lage (onshore/offshore) +- `ee_solar_taeglich`: Tägliche Solar-Ausbaudaten + +## Erweiterbarkeit + +Weitere Prozessoren können hinzugefügt werden, die auf denselben Daten in `mastr.db` arbeiten: +- `msr_biomasse_processor.py` +- `energiemix_processor.py` (ersetzt energiemix.py mit den Fraunhofer-Daten; erzeugt aktuelle Verlaufsdaten zum Energiemix) + +## Energiemix (`src/energiemix.py`) + +Monolithischer Scraper, der Daten des [Fraunhofer ISE](https://www.energy-charts.info/?l=de&c=DE) über die API holt, aufarbeitet und auf zwei Datawrapper-Grafiken schiebt. + +- **Chart "Erneuerbare-Anteil"** (`n3FOA`): Monatsmittel + Jahresdurchschnitte, 10 Jahre +- **Chart "Installierte Leistung"** (`p5sHV`): Kapazitäten nach Energieträger + +**Weshalb nicht aus dem MaStR?** Dort findet sich die *installierte* Leistung; was aus dieser Kapazität tatsächlich herauskommt, kann man erst im Nachhinein sagen - bzw. mit Modellen und unter Zuhilfenahme anderer Quellen ergänzen. Das tut die Fraunhofer-Plattform. + +## Einstiegspunkt + +`klimadashboard.py` wird vom zentralen Handler aufgerufen und orchestriert die einzelnen Module. diff --git a/ddj_cloud/scrapers/klimadashboard/__init__.py b/ddj_cloud/scrapers/klimadashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ddj_cloud/scrapers/klimadashboard/klimadashboard.py b/ddj_cloud/scrapers/klimadashboard/klimadashboard.py new file mode 100644 index 0000000..dcab33d --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/klimadashboard.py @@ -0,0 +1,73 @@ +from pathlib import Path + +import pandas as pd + +from ddj_cloud.scrapers.klimadashboard.src.energiemix import update_energiemix +from ddj_cloud.scrapers.klimadashboard.src.msr_dw_display import upload_all as upload_dw_charts +from ddj_cloud.scrapers.klimadashboard.src.msr_scraper import scrape_mastr +from ddj_cloud.scrapers.klimadashboard.src.msr_solar_processor import process_solar +from ddj_cloud.scrapers.klimadashboard.src.msr_wind_processor import process_wind +from ddj_cloud.utils.storage import ( + upload_dataframe, + upload_file, +) + +VERSION_STRING = "V0.05 vom 13.04.2026" + +# mastr.db in local_storage (analog zu anderen Scrapern) +DB_LOCAL_PATH = Path(__file__).parent.parent.parent.parent / "local_storage" / "klimadashboard" / "mastr.db" +DB_S3_KEY = "klimadashboard/mastr.db" + + +def _upload_db(): + """Lädt mastr.db auf S3 hoch.""" + if not DB_LOCAL_PATH.exists(): + print(" Warnung: mastr.db nicht gefunden, Upload übersprungen.") + return + upload_file( + DB_LOCAL_PATH.read_bytes(), + DB_S3_KEY, + archive=False, + ) + size_mb = DB_LOCAL_PATH.stat().st_size / 1024 / 1024 + print(f" mastr.db auf S3 hochgeladen ({size_mb:.1f} MB)") + + +def run(): + # Energiemix (Fraunhofer API) + df = update_energiemix() + upload_dataframe(df, "klimadashboard/test_energiemix1.csv") + + # MaStR: Scraper, Prozessoren, DB auf S3 + print("MaStR-Daten aktualisieren...") + DB_LOCAL_PATH.parent.mkdir(parents=True, exist_ok=True) + counts = scrape_mastr(DB_LOCAL_PATH) + total = sum(counts.values()) + print(f" MaStR-Scraper: {total} Einheiten geladen") + + # Wind + print("Wind-Daten verarbeiten...") + df_onshore, df_offshore, wind_summaries = process_wind(DB_LOCAL_PATH) + df_wind = pd.concat([df_onshore, df_offshore], ignore_index=True) + upload_dataframe(df_wind, "klimadashboard/wind_taeglich.csv") + for name, df_summary in wind_summaries.items(): + upload_dataframe(df_summary, f"klimadashboard/wind_{name}.csv") + + # Solar + print("Solar-Daten verarbeiten...") + df_solar, solar_summaries = process_solar(DB_LOCAL_PATH) + upload_dataframe(df_solar, "klimadashboard/solar_taeglich.csv") + for name, df_summary in solar_summaries.items(): + upload_dataframe(df_summary, f"klimadashboard/solar_{name}.csv") + + # Datawrapper-Charts aktualisieren + print("Datawrapper-Charts aktualisieren...") + upload_dw_charts( + wind_summaries=wind_summaries, + solar_summaries=solar_summaries, + ) + + # DB auf S3 hochladen + _upload_db() + + print("MaStR-Daten aktualisiert.") diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/Dokumentation MaStR Gesamtdatenexport.pdf b/ddj_cloud/scrapers/klimadashboard/msr_php/Dokumentation MaStR Gesamtdatenexport.pdf new file mode 100644 index 0000000..2f669ea Binary files /dev/null and b/ddj_cloud/scrapers/klimadashboard/msr_php/Dokumentation MaStR Gesamtdatenexport.pdf differ diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/Tabellenstrukturen.txt b/ddj_cloud/scrapers/klimadashboard/msr_php/Tabellenstrukturen.txt new file mode 100644 index 0000000..e1a69e5 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/Tabellenstrukturen.txt @@ -0,0 +1,85 @@ +-- +-- Tabellenstruktur für Tabelle `ee_wind` +-- (Tabelle mit allen WKA-Anlagen) +-- + +CREATE TABLE `ee_wind` ( + `mastrnr_einheit` varchar(32) NOT NULL, + `name_einheit` text DEFAULT NULL, + `betriebsstatus` text DEFAULT NULL, + `bruttoleistung` decimal(10,1) DEFAULT NULL, + `nettonennleistung` decimal(10,1) DEFAULT NULL, + `datum_inbetriebnahme` date DEFAULT NULL, + `datum_registrierung` date DEFAULT NULL, + `bundesland` text DEFAULT NULL, + `landkreis` mediumtext DEFAULT NULL, + `gemeinde` mediumtext DEFAULT NULL, + `plz` mediumtext DEFAULT NULL, + `ort` mediumtext DEFAULT NULL, + `strasse` mediumtext DEFAULT NULL, + `hausnummer` mediumtext DEFAULT NULL, + `gemarkung` mediumtext DEFAULT NULL, + `flurstueck` mediumtext DEFAULT NULL, + `gemeindeschluessel` int(11) DEFAULT NULL, + `breitengrad` decimal(10,6) DEFAULT NULL, + `laengengrad` decimal(10,6) DEFAULT NULL, + `name_windpark` mediumtext DEFAULT NULL, + `nabenhoehe` decimal(10,2) DEFAULT NULL, + `rotordurchmesser` decimal(10,2) DEFAULT NULL, + `hersteller_windanlage` mediumtext DEFAULT NULL, + `typenbezeichnung` mediumtext DEFAULT NULL, + `technologie` mediumtext DEFAULT NULL, + `lage_einheit` mediumtext DEFAULT NULL, + `letzte_aktualisierung` date DEFAULT NULL, + `datum_stilllegung` date DEFAULT NULL, + `datum_geplante_inbetriebnahme` date DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- +-- Indizes für die Tabelle `ee_wind` +-- +ALTER TABLE `ee_wind` + ADD KEY `idx_lage_einheit` (`lage_einheit`(17)), + ADD KEY `idx_betriebsstatus` (`betriebsstatus`(25)), + ADD KEY `idx_datum_inbetriebnahme` (`datum_inbetriebnahme`), + ADD KEY `idx_datum_stilllegung` (`datum_stilllegung`), + ADD KEY `idx_datum_geplante_inbetriebnahme` (`datum_geplante_inbetriebnahme`), + ADD KEY `idx_lage_status_datum_inbetriebnahme` (`lage_einheit`(17),`betriebsstatus`(25),`datum_inbetriebnahme`), + ADD KEY `idx_lage_status_datum_stilllegung` (`lage_einheit`(17),`betriebsstatus`(25),`datum_stilllegung`), + ADD KEY `idx_lage_status_datum_geplante_inbetriebnahme` (`lage_einheit`(17),`betriebsstatus`(25),`datum_geplante_inbetriebnahme`), + ADD KEY `idx_mastrnr` (`mastrnr_einheit`); +COMMIT; + + +------------------------------------------------------------------------------- + + +-- +-- Tabellenstruktur für Tabelle `ee_wind_taeglich` +-- (Tabelle, aus denen ich meine Ausgaben an die Diagramme generiere: Zeitliche Verläufe etc.) +-- + +CREATE TABLE `ee_wind_taeglich` ( + `datum` date NOT NULL, + `lage_einheit` text NOT NULL, + `installiert_gesamt` text NOT NULL, + `installiert_taeglich` text NOT NULL, + `geplant_gesamt` text NOT NULL, + `geplant_taeglich` text NOT NULL, + `noetig_gesamt` text NOT NULL, + `noetig_taeglich` text NOT NULL, + `stand` date NOT NULL, + `installiert_taeglich_wert` float NOT NULL, + `geplant_taeglich_wert` float NOT NULL, + `noetig_taeglich_wert` float NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin; + + +-- +-- Indizes für die Tabelle `ee_wind_taeglich` +-- +ALTER TABLE `ee_wind_taeglich` + ADD KEY `idx_datum` (`datum`), + ADD KEY `idx_lage_einheit` (`lage_einheit`(17)); +COMMIT; + diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/wka_daily.php b/ddj_cloud/scrapers/klimadashboard/msr_php/wka_daily.php new file mode 100644 index 0000000..64f23eb --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/wka_daily.php @@ -0,0 +1,164 @@ + + + + + WKA tägliche Aktualisierung + + + + +GetGefilterteListeStromErzeuger(array("apiKey" => $apikey, "marktakteurMastrNummer" => $marktakteurMastrNummer, "energietraeger" => "Wind", "datumAb" => $gestern, "limit" => $limit)); + + + echo ''; + echo ''; + $i = 0; + foreach ($result->Einheiten as $einheit) { + + $EinheitMastrNummer = $einheit->EinheitMastrNummer; + + $i++; + $query = "SELECT * FROM ee_wind WHERE mastrnr_einheit = '".$EinheitMastrNummer."' LIMIT 1"; + $result_sql = mysql_query($query); + if (mysql_affected_rows() > 0) {$vorhanden = "ja";} + else {$vorhanden = "nein";} + + + + IF ($vorhanden == "nein") { + $pause = mt_rand(2, 5); + // sleep($pause); + + $result2 = $client->GetEinheitWind(array("apiKey" => $apikey, "marktakteurMastrNummer" => $marktakteurMastrNummer, "einheitMastrNummer" => $EinheitMastrNummer)); + + + $Name = $result2->NameStromerzeugungseinheit; + $EinheitBetriebsstatus = $result2->EinheitBetriebsstatus; + $Bruttoleistung = $result2->Bruttoleistung; + $Nettonennleistung = $result2->Nettonennleistung; + $Inbetriebnahmedatum = $result2->Inbetriebnahmedatum; + $Registrierungsdatum = $result2->Registrierungsdatum; + $Bundesland = $result2->Bundesland; + $Landkreis = $result2->Landkreis; + $Gemeinde = $result2->Gemeinde; + $Postleitzahl = $result2->Postleitzahl; + $Ort = $result2->Ort; + $Strasse = $result2->Strasse; + $Gemarkung = $result2->Gemarkung; + $FlurFlurstuecknummern = $result2->FlurFlurstuecknummern; + $Gemeindeschluessel = $result2->Gemeindeschluessel; + $Breitengrad = $result2->Breitengrad; + $Laengengrad = $result2->Laengengrad; + $NameWindpark = $result2->NameWindpark; + $Nabenhoehe = $result2->Nabenhoehe; + $Rotordurchmesser = $result2->Rotordurchmesser; + $Hersteller = $result2->Hersteller->Wert; + $Typenbezeichnung = $result2->Typenbezeichnung; + $Technologie = $result2->Technologie; + $WindAnLandOderSee = $result2->WindAnLandOderSee; + $DatumLetzteAktualisierung = $result2->DatumLetzteAktualisierung; + if ($result2->DatumBeginnVoruebergehendeStilllegung > $result2->DatumEndgueltigeStilllegung) { + $Datum_Stilllegung = $result2->DatumBeginnVoruebergehendeStilllegung; + } + else { + $Datum_Stilllegung = $result2->DatumEndgueltigeStilllegung; + } + $GeplantesInbetriebnahmedatum = $result2->GeplantesInbetriebnahmedatum; + + + mysql_query ("INSERT INTO ee_wind (mastrnr_einheit, name_einheit, betriebsstatus, bruttoleistung, nettonennleistung, datum_inbetriebnahme, datum_registrierung, bundesland, landkreis, gemeinde, plz, ort, strasse, gemarkung, flurstueck, gemeindeschluessel, breitengrad, laengengrad, name_windpark, nabenhoehe, rotordurchmesser, hersteller_windanlage, typenbezeichnung, technologie, lage_einheit, letzte_aktualisierung, datum_stilllegung, datum_geplante_inbetriebnahme) VALUES ('$EinheitMastrNummer', '$Name', '$EinheitBetriebsstatus', '$Bruttoleistung', '$Nettonennleistung', '$Inbetriebnahmedatum', '$Registrierungsdatum', '$Bundesland', '$Landkreis', '$Gemeinde', '$Postleitzahl', '$Ort', '$Strasse', '$Gemarkung', '$FlurFlurstuecknummern', '$Gemeindeschluessel', '$Breitengrad', '$Laengengrad', '$NameWindpark', '$Nabenhoehe', '$Rotordurchmesser', '$Hersteller', '$Typenbezeichnung', '$Technologie', '$WindAnLandOderSee', '$DatumLetzteAktualisierung', '$Datum_Stilllegung', '$GeplantesInbetriebnahmedatum')"); + } + + + ELSE { + $pause = mt_rand(2, 5); + // sleep($pause); + + $result2 = $client->GetEinheitWind(array("apiKey" => $apikey, "marktakteurMastrNummer" => $marktakteurMastrNummer, "einheitMastrNummer" => $EinheitMastrNummer)); + + $Name = $result2->NameStromerzeugungseinheit; + $EinheitBetriebsstatus = $result2->EinheitBetriebsstatus; + $Bruttoleistung = $result2->Bruttoleistung; + $Nettonennleistung = $result2->Nettonennleistung; + $Inbetriebnahmedatum = $result2->Inbetriebnahmedatum; + $Registrierungsdatum = $result2->Registrierungsdatum; + $Bundesland = $result2->Bundesland; + $Landkreis = $result2->Landkreis; + $Gemeinde = $result2->Gemeinde; + $Postleitzahl = $result2->Postleitzahl; + $Ort = $result2->Ort; + $Strasse = $result2->Strasse; + $Gemarkung = $result2->Gemarkung; + $FlurFlurstuecknummern = $result2->FlurFlurstuecknummern; + $Gemeindeschluessel = $result2->Gemeindeschluessel; + $Breitengrad = $result2->Breitengrad; + $Laengengrad = $result2->Laengengrad; + $NameWindpark = $result2->NameWindpark; + $Nabenhoehe = $result2->Nabenhoehe; + $Rotordurchmesser = $result2->Rotordurchmesser; + $Hersteller = $result2->Hersteller->Wert; + $Typenbezeichnung = $result2->Typenbezeichnung; + $Technologie = $result2->Technologie; + $WindAnLandOderSee = $result2->WindAnLandOderSee; + $DatumLetzteAktualisierung = $result2->DatumLetzteAktualisierung; + if ($result2->DatumBeginnVoruebergehendeStilllegung > $result2->DatumEndgueltigeStilllegung) { + $Datum_Stilllegung = $result2->DatumBeginnVoruebergehendeStilllegung; + } + else { + $Datum_Stilllegung = $result2->DatumEndgueltigeStilllegung; + } + $GeplantesInbetriebnahmedatum = $result2->GeplantesInbetriebnahmedatum; + + + mysql_query ("UPDATE ee_wind SET name_einheit = '$Name', betriebsstatus = '$EinheitBetriebsstatus', bruttoleistung = '$Bruttoleistung', nettonennleistung = '$Nettonennleistung', datum_inbetriebnahme = '$Inbetriebnahmedatum', datum_registrierung = '$Registrierungsdatum', bundesland = '$Bundesland', landkreis = '$Landkreis', gemeinde = '$Gemeinde', plz = '$Postleitzahl', ort = '$Ort', strasse = '$Strasse', gemarkung = '$Gemarkung', flurstueck = '$FlurFlurstuecknummern', gemeindeschluessel = '$Gemeindeschluessel', breitengrad = '$Breitengrad', laengengrad = '$Laengengrad', name_windpark = '$NameWindpark', nabenhoehe = '$Nabenhoehe', rotordurchmesser = '$Rotordurchmesser', hersteller_windanlage = '$Hersteller', typenbezeichnung = '$Typenbezeichnung', technologie = '$Technologie', lage_einheit = '$WindAnLandOderSee', letzte_aktualisierung = '$DatumLetzteAktualisierung', datum_stilllegung = '$Datum_Stilllegung', datum_geplante_inbetriebnahme = '$GeplantesInbetriebnahmedatum' WHERE mastrnr_einheit = '$EinheitMastrNummer';"); + } + + + echo ''; + + ob_flush(); + flush(); + + } + echo '
Nr.MaStR-Nr.NameLageStandortBundeslandBruttoleistungStatusEintrag vorhanden?Inbetriebnahmedatum
'.$i.''.$EinheitMastrNummer.''.$Name.''.$WindAnLandOderSee.''.$Postleitzahl." ".$Ort.''.$Bundesland.''.$Bruttoleistung.''.$EinheitBetriebsstatus.''.$vorhanden.''.$Inbetriebnahmedatum.'
'; + print "
"; + print "
"; + + print "Mitteilungsfilter"; + + } catch (SoapFault $fault) { + echo "SOAP-Fehler: " . $fault->faultcode . ": " . $fault->faultstring; + } + + + ob_end_flush(); +?> + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/wka_to_data.php b/ddj_cloud/scrapers/klimadashboard/msr_php/wka_to_data.php new file mode 100644 index 0000000..4695aac --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/wka_to_data.php @@ -0,0 +1,221 @@ + + + + + WKA to Ausgabe-DB täglich + + + + +LageDatumBereits installierte LeistungZuwachsFest geplante zukünftige InbetriebnahmenZuwachsDurchschnittlich nötiger Ausbau für Klimaschutzziel 2030Zuwachs'; + + // VORHANDEN BIS 2009 + $vorleistung_bis_2009 = 0; + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft an Land' OR lage_einheit = 'WindAnLand') AND betriebsstatus <> 'In Planung' AND betriebsstatus <> 'InPlanung' AND betriebsstatus <> 'Vorübergehend stillgelegt' AND betriebsstatus <> 'VoruebergehendStillgelegt' AND datum_inbetriebnahme < '2010-01-01'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $vorleistung_bis_2009 = $vorleistung_bis_2009 + $row[nettonennleistung]; + } + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft an Land' OR lage_einheit = 'WindAnLand') AND datum_stilllegung <> '0000-00-00' AND datum_stilllegung < '2010-01-01'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $vorleistung_bis_2009 = $vorleistung_bis_2009 - $row[nettonennleistung]; + } + + $gesamtleistung_gw = $vorleistung_bis_2009 / 1000000; + $geplante_gesamtleistung_gw_land = 0; + $schon_virtuell_installiert = 0; + + // ZUBAU AB 2010 + $start = strtotime("2010-01-01"); + $ende = strtotime("2030-12-31"); + for ($i = $start; $i <= $ende; $i += 86400) { + $nettonennleistung_tag_mw = 0; + $abzug_tag_mw = 0; + $geplante_leistung_tag_mw = 0; + $datum = date("Y-m-d", $i); + + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft an Land' OR lage_einheit = 'WindAnLand') AND betriebsstatus <> 'In Planung' AND betriebsstatus <> 'InPlanung' AND betriebsstatus <> 'Vorübergehend stillgelegt' AND betriebsstatus <> 'VoruebergehendStillgelegt' AND datum_inbetriebnahme = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $nettonennleistung_tag_mw = $nettonennleistung_tag_mw + $row[nettonennleistung] / 1000; + } + + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft an Land' OR lage_einheit = 'WindAnLand') AND datum_stilllegung = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $abzug_tag_mw = $abzug_tag_mw + $row[nettonennleistung] / 1000; + } + + $gesamtleistung_gw = round ($gesamtleistung_gw + $nettonennleistung_tag_mw / 1000 - $abzug_tag_mw / 1000, 2); + $zuwachs = $nettonennleistung_tag_mw - $abzug_tag_mw; + $zuwachs_string = round ($nettonennleistung_tag_mw - $abzug_tag_mw, 1); + IF ($i == strtotime("2023-02-01")) {$stand_2023_02_01 = $gesamtleistung_gw;} + + IF ($datum >= $heute) { + // SCHON MIT DATUM GEPLANTE INSTALLATIONEN (NUR IN ZUKUNFT) + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft an Land' OR lage_einheit = 'WindAnLand') AND (betriebsstatus = 'In Planung' OR betriebsstatus = 'InPlanung') AND datum_geplante_inbetriebnahme = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $geplante_leistung_tag_mw = $geplante_leistung_tag_mw + $row[nettonennleistung] / 1000; + } + $geplante_gesamtleistung_gw_land = $geplante_gesamtleistung_gw_land + $geplante_leistung_tag_mw / 1000; + $summe_inbetrieb_und_geplant_land = round ($gesamtleistung_gw + $geplante_gesamtleistung_gw_land, 2); + } + + IF ($datum >= "2023-02-01") { + // DURCHSCHNITTLICH NÖTIGE INSTALLATIONEN (WIND-AN-LAND-GESETZ AM 1.2.2023 IN KRAFT GETRETEN), UM 115 GW ZU ERREICHEN + $noch_zu_installieren = 115 - $stand_2023_02_01; + $taeglich_noetige_leistung_gw_land = $noch_zu_installieren / $tage_bis_enddatum_land; + $noetige_gesamtleistung_gw_land = round ($stand_2023_02_01 + $schon_virtuell_installiert + $taeglich_noetige_leistung_gw_land, 2); + $schon_virtuell_installiert = $schon_virtuell_installiert + $taeglich_noetige_leistung_gw_land; + } + + + IF ($datum <= "2023-02-01") { + echo 'Windkraft an Land'.$datum.''.$gesamtleistung_gw.''.$zuwachs.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, installiert_gesamt, installiert_taeglich, stand, installiert_taeglich_wert) VALUES ('$datum', 'Windkraft an Land', '$gesamtleistung_gw', '$zuwachs_string', '$heute', '$zuwachs')"); + } + ELSEIF ($datum <= $heute) { + $noetig_taeglich = $taeglich_noetige_leistung_gw_land * 1000; + $noetig_taeglich_string = round ($noetig_taeglich, 1); + echo 'Windkraft an Land'.$datum.''.$gesamtleistung_gw.''.$zuwachs.''.$noetige_gesamtleistung_gw_land.''.$noetig_taeglich.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, installiert_gesamt, installiert_taeglich, noetig_gesamt, noetig_taeglich, stand, installiert_taeglich_wert, noetig_taeglich_wert) VALUES ('$datum', 'Windkraft an Land', '$gesamtleistung_gw', '$zuwachs_string', '$noetige_gesamtleistung_gw_land', '$noetig_taeglich_string', '$heute', '$zuwachs', '$noetig_taeglich')"); + } + ELSE { + $geplant_taeglich = $geplante_leistung_tag_mw; + $noetig_taeglich = $taeglich_noetige_leistung_gw_land * 1000; + $geplant_taeglich_string = round ($geplant_taeglich, 1); + $noetig_taeglich_string = round ($noetig_taeglich, 1); + echo 'Windkraft an Land'.$datum.''.$summe_inbetrieb_und_geplant_land.''.$geplant_taeglich.''.$noetige_gesamtleistung_gw_land.''.$noetig_taeglich.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, geplant_gesamt, geplant_taeglich, noetig_gesamt, noetig_taeglich, stand, geplant_taeglich_wert, noetig_taeglich_wert) VALUES ('$datum', 'Windkraft an Land', '$summe_inbetrieb_und_geplant_land', '$geplant_taeglich_string', '$noetige_gesamtleistung_gw_land', '$noetig_taeglich_string', '$heute', '$geplant_taeglich', '$noetig_taeglich')"); + } + + } + + + /////////////// + // OFF-SHORE // + /////////////// + + // VORHANDEN BIS 2009 + $vorleistung_bis_2009 = 0; + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft auf See' OR lage_einheit = 'WindAufSee') AND betriebsstatus <> 'In Planung' AND betriebsstatus <> 'InPlanung' AND betriebsstatus <> 'Vorübergehend stillgelegt' AND betriebsstatus <> 'VoruebergehendStillgelegt' AND datum_inbetriebnahme < '2010-01-01'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $vorleistung_bis_2009 = $vorleistung_bis_2009 + $row[nettonennleistung]; + } + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft auf See' OR lage_einheit = 'WindAufSee') AND datum_stilllegung <> '0000-00-00' AND datum_stilllegung < '2010-01-01'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $vorleistung_bis_2009 = $vorleistung_bis_2009 - $row[nettonennleistung]; + } + + $gesamtleistung_gw = $vorleistung_bis_2009 / 1000000; + $geplante_gesamtleistung_gw_see = 0; + $schon_virtuell_installiert = 0; + + // ZUBAU AB 2010 + $start = strtotime("2010-01-01"); + $ende = strtotime("2030-12-31"); + for ($i = $start; $i <= $ende; $i += 86400) { + $nettonennleistung_tag_mw = 0; + $abzug_tag_mw = 0; + $geplante_leistung_tag_mw = 0; + $datum = date("Y-m-d", $i); + + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft auf See' OR lage_einheit = 'WindAufSee') AND betriebsstatus <> 'In Planung' AND betriebsstatus <> 'InPlanung' AND betriebsstatus <> 'Vorübergehend stillgelegt' AND betriebsstatus <> 'VoruebergehendStillgelegt' AND datum_inbetriebnahme = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $nettonennleistung_tag_mw = $nettonennleistung_tag_mw + $row[nettonennleistung] / 1000; + } + + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft auf See' OR lage_einheit = 'WindAufSee') AND datum_stilllegung = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $abzug_tag_mw = $abzug_tag_mw + $row[nettonennleistung] / 1000; + } + + $gesamtleistung_gw = round ($gesamtleistung_gw + $nettonennleistung_tag_mw / 1000 - $abzug_tag_mw / 1000, 2); + $zuwachs = $nettonennleistung_tag_mw - $abzug_tag_mw; + $zuwachs_string = round ($nettonennleistung_tag_mw - $abzug_tag_mw, 1); + IF ($i == strtotime("2023-01-01")) {$stand_2023_01_01 = $gesamtleistung_gw;} + + IF ($datum >= $heute) { + // SCHON MIT DATUM GEPLANTE INSTALLATIONEN (NUR IN ZUKUNFT) + $query = "SELECT nettonennleistung FROM ee_wind WHERE (lage_einheit = 'Windkraft auf See' OR lage_einheit = 'WindAufSee') AND (betriebsstatus = 'In Planung' OR betriebsstatus = 'InPlanung') AND datum_geplante_inbetriebnahme = '".$datum."'"; + $result = mysql_query($query); + while ($row=mysql_fetch_array($result)){ + $geplante_leistung_tag_mw = $geplante_leistung_tag_mw + $row[nettonennleistung] / 1000; + } + $geplante_gesamtleistung_gw_see = $geplante_gesamtleistung_gw_see + $geplante_leistung_tag_mw / 1000; + $summe_inbetrieb_und_geplant_see = round ($gesamtleistung_gw + $geplante_gesamtleistung_gw_see, 2); + } + + IF ($datum >= "2023-01-01") { + // DURCHSCHNITTLICH NÖTIGE INSTALLATIONEN (WIND-AN-SEE-GESETZ AM 1.1.2023 IN KRAFT GETRETEN), UM 30 GW ZU ERREICHEN + $noch_zu_installieren = 30 - $stand_2023_01_01; + $taeglich_noetige_leistung_gw_see = $noch_zu_installieren / $tage_bis_enddatum_see; + $noetige_gesamtleistung_gw_see = round ($stand_2023_01_01 + $schon_virtuell_installiert + $taeglich_noetige_leistung_gw_see, 2); + $schon_virtuell_installiert = $schon_virtuell_installiert + $taeglich_noetige_leistung_gw_see; + } + + + IF ($datum <= "2023-01-01") { + echo 'Windkraft auf See'.$datum.''.$gesamtleistung_gw.''.$zuwachs.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, installiert_gesamt, installiert_taeglich, stand, installiert_taeglich_wert) VALUES ('$datum', 'Windkraft auf See', '$gesamtleistung_gw', '$zuwachs_string', '$heute', '$zuwachs')"); + } + ELSEIF ($datum <= $heute) { + $noetig_taeglich = $taeglich_noetige_leistung_gw_see * 1000; + $noetig_taeglich_string = round ($noetig_taeglich, 1); + echo 'Windkraft auf See'.$datum.''.$gesamtleistung_gw.''.$zuwachs.''.$noetige_gesamtleistung_gw_see.''.$noetig_taeglich.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, installiert_gesamt, installiert_taeglich, noetig_gesamt, noetig_taeglich, stand, installiert_taeglich_wert, noetig_taeglich_wert) VALUES ('$datum', 'Windkraft auf See', '$gesamtleistung_gw', '$zuwachs_string', '$noetige_gesamtleistung_gw_see', '$noetig_taeglich_string', '$heute', '$zuwachs', '$noetig_taeglich')"); + } + ELSE { + $geplant_taeglich = $geplante_leistung_tag_mw; + $noetig_taeglich = $taeglich_noetige_leistung_gw_see * 1000; + $geplant_taeglich_string = round ($geplant_taeglich, 1); + $noetig_taeglich_string = round ($noetig_taeglich, 1); + echo 'Windkraft auf See'.$datum.''.$summe_inbetrieb_und_geplant_see.''.$geplant_taeglich.''.$noetige_gesamtleistung_gw_see.''.$noetig_taeglich.''; + mysql_query ("INSERT INTO ee_wind_taeglich (datum, lage_einheit, geplant_gesamt, geplant_taeglich, noetig_gesamt, noetig_taeglich, stand, geplant_taeglich_wert, noetig_taeglich_wert) VALUES ('$datum', 'Windkraft auf See', '$summe_inbetrieb_und_geplant_see', '$geplant_taeglich_string', '$noetige_gesamtleistung_gw_see', '$noetig_taeglich_string', '$heute', '$geplant_taeglich', '$noetig_taeglich')"); + } + + } + + echo ''; + + + +?> + + + diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegBiomasse.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegBiomasse.xsd new file mode 100644 index 0000000..2fa967c --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegBiomasse.xsd @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegGeothermieGrubengasDruckentspannung.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegGeothermieGrubengasDruckentspannung.xsd new file mode 100644 index 0000000..ef4db15 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegGeothermieGrubengasDruckentspannung.xsd @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSolar.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSolar.xsd new file mode 100644 index 0000000..c385058 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSolar.xsd @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSpeicher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSpeicher.xsd new file mode 100644 index 0000000..f1d4849 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegSpeicher.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWasser.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWasser.xsd new file mode 100644 index 0000000..23d229a --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWasser.xsd @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWind.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWind.xsd new file mode 100644 index 0000000..1d7d504 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenEegWind.xsd @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenGasSpeicher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenGasSpeicher.xsd new file mode 100644 index 0000000..22d0c07 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenGasSpeicher.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenKwk.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenKwk.xsd new file mode 100644 index 0000000..a742e46 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenKwk.xsd @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenStromSpeicher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenStromSpeicher.xsd new file mode 100644 index 0000000..dd2eb37 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/AnlagenStromSpeicher.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Bilanzierungsgebiete.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Bilanzierungsgebiete.xsd new file mode 100644 index 0000000..ae1ad69 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Bilanzierungsgebiete.xsd @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenAenderungNetzbetreiberzuordnungen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenAenderungNetzbetreiberzuordnungen.xsd new file mode 100644 index 0000000..6868a11 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenAenderungNetzbetreiberzuordnungen.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenBiomasse.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenBiomasse.xsd new file mode 100644 index 0000000..88f2499 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenBiomasse.xsd @@ -0,0 +1,319 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasErzeuger.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasErzeuger.xsd new file mode 100644 index 0000000..0da4395 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasErzeuger.xsd @@ -0,0 +1,159 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasSpeicher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasSpeicher.xsd new file mode 100644 index 0000000..9eff033 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasSpeicher.xsd @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasverbraucher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasverbraucher.xsd new file mode 100644 index 0000000..47c4eae --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGasverbraucher.xsd @@ -0,0 +1,155 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGenehmigung.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGenehmigung.xsd new file mode 100644 index 0000000..a870f16 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGenehmigung.xsd @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGeothermieGrubengasDruckentspannung.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGeothermieGrubengasDruckentspannung.xsd new file mode 100644 index 0000000..c194659 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenGeothermieGrubengasDruckentspannung.xsd @@ -0,0 +1,265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenKernkraft.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenKernkraft.xsd new file mode 100644 index 0000000..130f9ae --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenKernkraft.xsd @@ -0,0 +1,252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenSolar.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenSolar.xsd new file mode 100644 index 0000000..afc811c --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenSolar.xsd @@ -0,0 +1,350 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromSpeicher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromSpeicher.xsd new file mode 100644 index 0000000..e46fd4a --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromSpeicher.xsd @@ -0,0 +1,365 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromVerbraucher.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromVerbraucher.xsd new file mode 100644 index 0000000..f578da5 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenStromVerbraucher.xsd @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenVerbrennung.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenVerbrennung.xsd new file mode 100644 index 0000000..0989091 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenVerbrennung.xsd @@ -0,0 +1,450 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWasser.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWasser.xsd new file mode 100644 index 0000000..71602c6 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWasser.xsd @@ -0,0 +1,283 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWind.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWind.xsd new file mode 100644 index 0000000..5b98a7a --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/EinheitenWind.xsd @@ -0,0 +1,383 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Einheitentypen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Einheitentypen.xsd new file mode 100644 index 0000000..4f07c14 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Einheitentypen.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Ertuechtigungen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Ertuechtigungen.xsd new file mode 100644 index 0000000..1af9c11 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Ertuechtigungen.xsd @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteEinheiten.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteEinheiten.xsd new file mode 100644 index 0000000..13a3313 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteEinheiten.xsd @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteMarktakteure.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteMarktakteure.xsd new file mode 100644 index 0000000..6ab446e --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/GeloeschteUndDeaktivierteMarktakteure.xsd @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogkategorien.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogkategorien.xsd new file mode 100644 index 0000000..5166806 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogkategorien.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogwerte.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogwerte.xsd new file mode 100644 index 0000000..1be2b75 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Katalogwerte.xsd @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationen.xsd new file mode 100644 index 0000000..15a9c25 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationen.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationstypen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationstypen.xsd new file mode 100644 index 0000000..297b103 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Lokationstypen.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktakteure.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktakteure.xsd new file mode 100644 index 0000000..744c06f --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktakteure.xsd @@ -0,0 +1,1191 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/MarktakteureUndRollen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/MarktakteureUndRollen.xsd new file mode 100644 index 0000000..3638199 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/MarktakteureUndRollen.xsd @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktfunktionen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktfunktionen.xsd new file mode 100644 index 0000000..8ecebc5 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktfunktionen.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktrollen.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktrollen.xsd new file mode 100644 index 0000000..1d1bfd1 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Marktrollen.xsd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netzanschlusspunkte.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netzanschlusspunkte.xsd new file mode 100644 index 0000000..1e90528 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netzanschlusspunkte.xsd @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netze.xsd b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netze.xsd new file mode 100644 index 0000000..773331b --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/msr_php/xsd/Netze.xsd @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/src/TODO.md b/ddj_cloud/scrapers/klimadashboard/src/TODO.md new file mode 100644 index 0000000..4580344 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/TODO.md @@ -0,0 +1,38 @@ +# Energiemix als Demo für das Energie-Dashboard + +## Anteil erneuerbare Energien + +Die einfachste Grafik: Das ist ein Prozentanteil, der über die Fraunhofer-API taggenau abrufbar ist. Die Daten gehen bis zum 1.1.2015 zurück. + +- Einfaches Datenauslese-Skript ✅ +- Upload Datawrapper ✅ +- Scraper im wdr-data Format +- Scraper im mage.ai Format +- Einbau in Quarks-Demoseite + +## Energiemix + +- Recherche zu den Daten +- Datenauslese-Skript +- Upload Datawrapper +- Scraperformat festlegen + +## MaStR-Daten (Marktstammdatenregister) + +### Scraper ✅ +- open-mastr Bulk-Download aller Energiearten ✅ +- Isoliertes venv via PEP 723 / uv run ✅ + +### Wind-Prozessor ✅ +- Tägliche Ausbaudaten (onshore/offshore) ✅ +- Monatliche/jährliche Zusammenfassungen ✅ +- Klimaziel-Berechnung (115 GW onshore, 30 GW offshore) ✅ + +### Solar-Prozessor ✅ +- Tägliche Ausbaudaten ✅ +- Monatliche/jährliche Zusammenfassungen ✅ +- Klimaziel-Berechnung (215 GW) ✅ + +### Offen +- Datawrapper-Charts für Wind und Solar erstellen +- Biomasse-Prozessor \ No newline at end of file diff --git a/ddj_cloud/scrapers/klimadashboard/src/energiemix.py b/ddj_cloud/scrapers/klimadashboard/src/energiemix.py new file mode 100644 index 0000000..eb4c99f --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/energiemix.py @@ -0,0 +1,238 @@ +""" +Holt den täglichen Anteil erneuerbarer Energien von der Fraunhofer Energy Charts API +und schreibt die Daten auf eine Datawrapper-Chart (Flächengrafik). +""" + +import os # noqa: I001 +from datetime import datetime + +import pandas as pd +import datawrapper as dw +import requests +import sentry_sdk +from dotenv import load_dotenv + +load_dotenv() + +FRAUNHOFER_URL = "https://api.energy-charts.info/" +DAILY_SHARE = "ren_share_daily_avg" +SHARE_FORECAST = "ren_share_forecast" +PUBLIC_POWER = "public_power" # Stromerzeugungs-Kapazität (öffentlich) +INSTALLED_POWER = "installed_power" +# PRICES = "price" # Spotmarkt-Preise + +MIX_ID = "n3FOA" +MIX_NOTES = '
quarks.deDurchschnittlicher Anteil von Wind, Photovoltaik, Wasserkraft und Biomasse im Strommix.' +POWER_ID = "p5sHV" +POWER_NOTES = '
quarks.deInstallierte Kapazitäten.' + +def fetch_renewable_share(year=None): + """Holt die täglichen Erneuerbare-Anteile für ein bestimmtes Jahr.""" + if not year: + year = datetime.now().year + try: + resp = requests.get(FRAUNHOFER_URL + DAILY_SHARE, params={"country": "de", "year": year}) + resp.raise_for_status() + except requests.RequestException as e: + sentry_sdk.capture_exception(e) + raise + data = resp.json() + return data["days"], data["data"] + + +def fetch_renewable_forecast(): + """Vorhersage. Umfasst idR nur den nächsten Tag in Viertelstundenschritten.""" + try: + resp = requests.get(FRAUNHOFER_URL + SHARE_FORECAST, params={"country": "de"}) + resp.raise_for_status() + except requests.RequestException as e: + sentry_sdk.capture_exception(e) + raise + data = resp.json() + # Unix-Sekunden in Daten umwandeln + dates = [datetime.fromtimestamp(d).strftime("%d.%m.%Y %H:%M") for d in data["unix_seconds"]] + return dates, data["ren_share"] + + +def fetch_public_power(): + """ + Holt die gesamte Stromproduktion für die letzten 24 Stunden. + Die Daten sind in 15-Minuten-Schritten. + """ + try: + resp = requests.get(FRAUNHOFER_URL + PUBLIC_POWER, params={"country": "de"}) + resp.raise_for_status() + except requests.RequestException as e: + sentry_sdk.capture_exception(e) + raise + data = resp.json() + data_df = pd.DataFrame({"time": data["unix_seconds"]}).set_index("time") + for item in data["production_types"]: + data_df[item["name"]] = item["data"] + return data_df + + +def fetch_installed_power(time_step="yearly"): + """Holt die installierte Leistung für die letzten 24 Stunden.""" + try: + resp = requests.get( + FRAUNHOFER_URL + INSTALLED_POWER, params={"country": "de", "time_step": time_step} + ) + resp.raise_for_status() + except requests.RequestException as e: + sentry_sdk.capture_exception(e) + raise + data = resp.json() + data_df = pd.DataFrame({"time": data["time"]}).set_index("time") + for item in data["production_types"]: + data_df[item["name"]] = item["data"] + return data_df + + +def fetch_last_n_years(n=10): + """Holt die täglichen Erneuerbare-Anteile der letzten n Jahre.""" + current_year = datetime.now().year + all_days = [] + all_values = [] + for year in range(current_year - n + 1, current_year + 1): + print(f" Lade {year}...") + days, values = fetch_renewable_share(year) + all_days.extend(days) + all_values.extend(values) + return all_days, all_values + + +def build_dataframe(days, values): + """Baut ein DataFrame aus den Rohdaten.""" + df = pd.DataFrame({"Datum": days, "Anteil Erneuerbare (%)": values}) + df["Datum"] = pd.to_datetime(df["Datum"], format="%d.%m.%Y") + df = df.dropna(subset=["Anteil Erneuerbare (%)"]) + return df + + +def aggregate_monthly(df): + """Aggregiert die täglichen Werte zu Monatsmittelwerten.""" + df_monthly = df.resample("MS", on="Datum").agg({"Anteil Erneuerbare (%)": "mean"}) + df_monthly = df_monthly.reset_index() + df_monthly["Anteil Erneuerbare (%)"] = df_monthly["Anteil Erneuerbare (%)"].round(1) + return df_monthly + + +def aggregate_yearly(df): + """Berechnet Jahresdurchschnitte, jeweils auf den 1. Dezember des Jahres gelegt.""" + df_yearly = df.groupby(df["Datum"].dt.year).agg({"Anteil Erneuerbare (%)": "mean"}) + df_yearly = df_yearly.reset_index() + df_yearly.columns = ["Jahr", "Jahresdurchschnitt (%)"] + df_yearly["Jahresdurchschnitt (%)"] = df_yearly["Jahresdurchschnitt (%)"].round(1) + df_yearly["Datum"] = pd.to_datetime(df_yearly["Jahr"].astype(str) + "-12-01") + df_yearly = df_yearly.drop(columns=["Jahr"]) + return df_yearly + + +def build_csv(df): + """Konvertiert das DataFrame in CSV für Datawrapper.""" + df = df.copy() + df["Datum"] = df["Datum"].dt.strftime("%Y-%m-%d") + return df.to_csv(index=False) + + +def build_csv_from_index(df): + """Konvertiert ein DataFrame mit Index als Zeitachse in CSV für Datawrapper.""" + return df.to_csv() + + +def dw_authenticate(): + token = os.environ.get("DW_API_KEY_JE") + if not token: + raise RuntimeError("Bitte DW_API_KEY_JE für Datawrapper als Umgebungsvariable setzen.") + client = dw.Datawrapper(access_token = token) + return client + + +def upload_to_datawrapper(dw_client, dw_id, csv_data, metadata = None): + # Update the chart + dw_client.add_data(chart_id=dw_id, data=csv_data) + if metadata: + dw_client.update_metadata( + chart_id=dw_id, + metadata=metadata + ) + """ + # The usual suspects: + metadata = { + title="Demo", + intro="This chart shows population trends over the past decade.", + notes="Data updated quarterly.", + source_name="Destatis", + source_url="https://www.destatis.de", + byline="WDR-Data", + "visualize": { + "custom-colors": { + "Category A": "#FF6B6B", + "Category B": "#4ECDC4", + "Category C": "#45B7D1" + } + } + You may also save the metadata as a JSON for remote control. + """ + # Republish to see changes + dw_client.publish_chart(chart_id=dw_id) + print(f"Chart publiziert: https://datawrapper.dwcdn.net/{dw_id}/") + + +def update_energiemix(): + print("Datawrapper einbinden...") + dw_client = dw_authenticate() + + + + print("Hole Daten der letzten 10 Jahre von Fraunhofer Energy Charts...") + days, values = fetch_last_n_years(10) + print(f"{len(days)} Tage geladen.") + + df = build_dataframe(days, values) + df_monthly = aggregate_monthly(df) + print(f"{len(df_monthly)} Monate aggregiert.") + + df_yearly = aggregate_yearly(df) + print(f"{len(df_yearly)} Jahresdurchschnitte berechnet.") + + # Beide Reihen über Datum zusammenführen + df_combined = pd.merge(df_monthly, df_yearly, on="Datum", how="left") + csv_data = build_csv(df_combined) + print("CSV erstellt.") + metadata = { "annotate": { + "notes": f"{MIX_NOTES}

Zuletzt aktualisiert: {datetime.now().strftime('%d.%m.%Y, %H:%M')}", + } + } + upload_to_datawrapper(dw_client, MIX_ID, csv_data, metadata) + + installed_df = fetch_installed_power() + # Daten filtern + kapa_df = installed_df[[ + "Nuclear", + "Fossil brown coal / lignite", + "Fossil hard coal", + "Fossil gas", + "Fossil oil", + "Other, non-renewable", + "Hydro", + "Biomass", + "Wind offshore", + "Wind onshore", + "Solar DC", + "Solar AC", + ]] + kapa_csv = build_csv_from_index(kapa_df) + metadata = { "annotate": { + "notes": f"{POWER_NOTES}

Zuletzt aktualisiert: {datetime.now().strftime('%d.%m.%Y, %H:%M')}", + } + } + upload_to_datawrapper(dw_client, POWER_ID, kapa_csv, metadata) + + + return df_combined + + +if __name__ == "__main__": + update_energiemix() diff --git a/ddj_cloud/scrapers/klimadashboard/src/msr_dw_display.py b/ddj_cloud/scrapers/klimadashboard/src/msr_dw_display.py new file mode 100644 index 0000000..004e14f --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/msr_dw_display.py @@ -0,0 +1,108 @@ +""" +Datawrapper-Upload für Wind- und Solar-Ausbaudaten. + +Nimmt die von msr_wind_processor und msr_solar_processor erzeugten DataFrames +und lädt sie gefiltert in Datawrapper-Charts hoch. + +Charts (alle monatlich): +- Wind-Ausbau: Gesamtleistung onshore/offshore + nötig +- Solar-Ausbau: Gesamtleistung + nötig +- Wind-Zubau: Neue Kapazität onshore/offshore pro Monat +- Solar-Zubau: Neue Kapazität pro Monat +""" + +import os +from datetime import datetime + +import pandas as pd +from datawrapper import Datawrapper +from dotenv import load_dotenv + +load_dotenv() + +# Jährliche statt monatliche Aggregation +YEARLY_AGGREGATES = True + +# String für die Anmerkungen +NOTES_STR = '
quarks.de' + +# Datawrapper Chart-IDs (TODO: eintragen nach Erstellung) +CHART_WIND = "EgOti" # Wind-Ausbau Gesamtleistung +CHART_SOLAR = "1rxLQ" # Solar-Ausbau Gesamtleistung +CHART_WIND_ZUBAU = "7yMTK" # Wind-Zubau +CHART_SOLAR_ZUBAU = "kPzGf" # Solar-Zubau + + +def _get_dw_client() -> Datawrapper: + token = os.environ.get("DW_API_KEY_JE") + if not token: + msg = "Bitte DW_API_KEY_JE als Umgebungsvariable mit Datawrapper-Key setzen." + raise RuntimeError(msg) + return Datawrapper(access_token=token) + + +def _upload_chart(client: Datawrapper, chart_id: str, df: pd.DataFrame, title: str): + """Lädt DataFrame als CSV in einen Datawrapper-Chart hoch.""" + if not chart_id: + print(f" Überspringe '{title}' (keine Chart-ID konfiguriert)") + return + # String mit der letzten Aktualisierung + notes_str = f"{NOTES_STR}Zuletzt aktualisiert: {datetime.now().strftime('%d.%m.%Y, %H:%M')}" + csv_data = df.to_csv(index=False) + client.add_data(chart_id=chart_id, data=csv_data) + client.update_chart(chart_id=chart_id, title=title, metadata={"annotate": {"notes": notes_str}}) + client.publish_chart(chart_id=chart_id) + print(f" Chart '{title}' aktualisiert ({chart_id})") + + +def build_wind_chart_data(wind_gesamt_monatlich: pd.DataFrame) -> pd.DataFrame: + """Wind-Gesamtleistung monatlich: Onshore, Offshore, geplant, nötig.""" + df = wind_gesamt_monatlich.copy() + df.columns = ["Datum", "Onshore (GW)", "Onshore geplant (GW)", "Offshore (GW)", "Offshore geplant (GW)"] + return df + + +def build_solar_chart_data(solar_gesamt_monatlich: pd.DataFrame) -> pd.DataFrame: + """Solar-Gesamtleistung monatlich: Installiert, geplant.""" + df = solar_gesamt_monatlich.copy() + df.columns = ["Datum", "Installiert (GW)", "Geplant (GW)"] + return df + + +def build_wind_zubau_data(wind_zubau_monatlich: pd.DataFrame) -> pd.DataFrame: + """Wind-Zubau monatlich: Onshore, Offshore, geplant.""" + df = wind_zubau_monatlich.copy() + df.columns = ["Datum", "Onshore (MW)", "Onshore geplant (MW)", "Offshore (MW)", "Offshore geplant (MW)"] + return df + + +def build_solar_zubau_data(solar_zubau_monatlich: pd.DataFrame) -> pd.DataFrame: + """Solar-Zubau monatlich: Installiert, geplant.""" + df = solar_zubau_monatlich.copy() + df.columns = ["Datum", "Installiert (MW)", "Geplant (MW)"] + return df + + +def upload_all( + wind_summaries: dict[str, pd.DataFrame], + solar_summaries: dict[str, pd.DataFrame], +): + """Lädt alle Charts auf Datawrapper hoch. + + Nutzt monatliche oder jährliche Daten je nach YEARLY_AGGREGATES. + """ + suffix = "jaehrlich" if YEARLY_AGGREGATES else "monatlich" + period = "pro Jahr" if YEARLY_AGGREGATES else "pro Monat" + client = _get_dw_client() + + df = build_wind_chart_data(wind_summaries[f"gesamt_{suffix}"]) + _upload_chart(client, CHART_WIND, df, "Windkraft-Ausbau in Deutschland") + + df = build_solar_chart_data(solar_summaries[f"gesamt_{suffix}"]) + _upload_chart(client, CHART_SOLAR, df, "Solarenergie-Ausbau in Deutschland") + + df = build_wind_zubau_data(wind_summaries[f"zubau_{suffix}"]) + _upload_chart(client, CHART_WIND_ZUBAU, df, f"Windkraft-Zubau {period}") + + df = build_solar_zubau_data(solar_summaries[f"zubau_{suffix}"]) + _upload_chart(client, CHART_SOLAR_ZUBAU, df, f"Solarenergie-Zubau {period}") diff --git a/ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py b/ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py new file mode 100644 index 0000000..6bb7afc --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py @@ -0,0 +1,102 @@ +""" +MaStR-Scraper: Lädt alle Energiearten aus dem Marktstammdatenregister +über die open-mastr-Bibliothek (Bulk-Download, kein API-Key nötig). + +Schreibt die Daten in eine lokale SQLite-Datenbank (mastr.db). +""" + +import sqlite3 +from datetime import date +from pathlib import Path + +import pandas as pd +import sentry_sdk +from open_mastr import Mastr + +ENERGY_TYPES = ["wind", "solar", "biomass", "hydro", "combustion", "nuclear", "gsgk", "storage"] + +OPEN_MASTR_DB = Path.home() / ".open-MaStR" / "data" / "sqlite" / "open-mastr.db" + + +def _check_last_download(target_db: Path) -> str | None: + """Prüft DatumDownload in der Ziel-DB. Gibt Datum als String oder None zurück.""" + if not target_db.exists(): + return None + try: + with sqlite3.connect(target_db) as conn: + row = conn.execute("SELECT DatumDownload FROM wind_extended LIMIT 1").fetchone() + return row[0] if row else None + except Exception: + return None + + +def scrape_mastr(db_path: Path) -> dict[str, int]: + """Lädt alle Energiearten via open-mastr und schreibt sie in die lokale DB. + + Überspringt den Download wenn die DB bereits Daten von heute enthält. + + Args: + db_path: Pfad zur Ziel-SQLite-Datenbank (mastr.db) + + Returns: + Dict mit Anzahl der Einheiten pro Energieart. + """ + # Skip download if DB already has today's data + last_download = _check_last_download(db_path) + today = date.today().isoformat() + + if last_download == today: + print(f" DB bereits aktuell ({today}), überspringe Download.") + counts = {} + with sqlite3.connect(db_path) as conn: + for energy_type in ENERGY_TYPES: + table_name = f"{energy_type}_extended" + try: + row = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone() # noqa: S608 + counts[energy_type] = row[0] + except Exception: + counts[energy_type] = 0 + return counts + + print(f" Letzter Download: {last_download or 'keiner'}, starte Update...") + + try: + mastr = Mastr() + mastr.download(data=ENERGY_TYPES) + except Exception as e: + sentry_sdk.capture_exception(e) + raise + + counts = {} + total = len(ENERGY_TYPES) + + with sqlite3.connect(OPEN_MASTR_DB) as src_conn, sqlite3.connect(db_path) as dst_conn: + for i, energy_type in enumerate(ENERGY_TYPES, 1): + table_name = f"{energy_type}_extended" + print(f" [{i}/{total}] {energy_type}...") + + try: + df = pd.read_sql(f"SELECT * FROM {table_name}", src_conn) # noqa: S608 + except Exception as e: + print(f" Warnung: Tabelle {table_name} nicht gefunden: {e}") + sentry_sdk.capture_exception(e) + continue + + col_temp = "DatumBeginnVoruebergehendeStilllegung" + col_final = "DatumEndgueltigeStilllegung" + if col_temp in df.columns and col_final in df.columns: + temp = pd.to_datetime(df[col_temp], errors="coerce") + final = pd.to_datetime(df[col_final], errors="coerce") + combined = pd.concat([temp, final], axis=1).max(axis=1) + df["datum_stilllegung"] = combined.dt.strftime("%Y-%m-%d") + df.loc[combined.isna(), "datum_stilllegung"] = None + + df.to_sql(table_name, dst_conn, if_exists="replace", index=False) + counts[energy_type] = len(df) + print(f" [{i}/{total}] {energy_type}: {len(df)} Einheiten") + + return counts + + +if __name__ == "__main__": + scrape_mastr(Path(__file__).parent / "mastr.db") diff --git a/ddj_cloud/scrapers/klimadashboard/src/msr_solar_processor.py b/ddj_cloud/scrapers/klimadashboard/src/msr_solar_processor.py new file mode 100644 index 0000000..ce4fcfe --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/msr_solar_processor.py @@ -0,0 +1,261 @@ +""" +Solar-Prozessor: Berechnet tägliche Ausbaudaten (installiert, geplant, nötig) +aus der lokalen MaStR-Datenbank. + +Liest aus solar_extended, schreibt in ee_solar_taeglich. +Analog zu msr_wind_processor.py. +""" + +import sqlite3 +from datetime import datetime +from pathlib import Path + +import pandas as pd +import sentry_sdk + +# Ausbauziel 2030 +TARGET_SOLAR_GW = 215 # EEG 2023 +TARGET_DATE = "2031-01-01" +BASELINE_DATE = "2023-01-01" # EEG 2023 in Kraft + +# Status-Filter für inaktive Anlagen +INACTIVE_STATUSES = ("In Planung", "Vorübergehend stillgelegt") + + +def _create_result_table(db: sqlite3.Connection): + """Erstellt die Ergebnis-Tabelle ee_solar_taeglich.""" + db.execute(""" + CREATE TABLE IF NOT EXISTS ee_solar_taeglich ( + datum TEXT NOT NULL PRIMARY KEY, + installiert_gesamt REAL, + installiert_taeglich REAL, + geplant_gesamt REAL, + geplant_taeglich REAL, + noetig_gesamt REAL, + noetig_taeglich REAL, + stand TEXT NOT NULL + ) + """) + db.commit() + + +def _calculate_daily_capacity(df: pd.DataFrame) -> pd.DataFrame: + """Berechnet tägliche Ausbaudaten für Solarenergie.""" + heute = datetime.now().strftime("%Y-%m-%d") + + # Aktive Anlagen filtern + active = df[~df["EinheitBetriebsstatus"].isin(INACTIVE_STATUSES)].copy() + active["Inbetriebnahmedatum"] = pd.to_datetime(active["Inbetriebnahmedatum"]) + active["datum_stilllegung"] = pd.to_datetime(active["datum_stilllegung"]) + active["Nettonennleistung"] = pd.to_numeric(active["Nettonennleistung"], errors="coerce") + + # Geplante Anlagen + planned = df[df["EinheitBetriebsstatus"] == "In Planung"].copy() + planned["GeplantesInbetriebnahmedatum"] = pd.to_datetime( + planned["GeplantesInbetriebnahmedatum"] + ) + planned["Nettonennleistung"] = pd.to_numeric(planned["Nettonennleistung"], errors="coerce") + + # Vorleistung bis Ende 2009 + pre_2010 = active[active["Inbetriebnahmedatum"] < "2010-01-01"]["Nettonennleistung"].sum() + decom_pre_2010 = active[ + active["datum_stilllegung"].notna() & (active["datum_stilllegung"] < "2010-01-01") + ]["Nettonennleistung"].sum() + base_capacity_kw = pre_2010 - decom_pre_2010 + + # Zubau pro Tag (kW) + additions = ( + active[active["Inbetriebnahmedatum"] >= "2010-01-01"] + .groupby("Inbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + # Abbau pro Tag (kW) + removals = ( + active[active["datum_stilllegung"].notna() & (active["datum_stilllegung"] >= "2010-01-01")] + .groupby("datum_stilllegung")["Nettonennleistung"] + .sum() + ) + # Geplante Zubauten pro Tag (kW) + planned_additions = ( + planned[planned["GeplantesInbetriebnahmedatum"].notna()] + .groupby("GeplantesInbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + + baseline_dt = pd.Timestamp(BASELINE_DATE) + target_dt = pd.Timestamp(TARGET_DATE) + days_to_target = (target_dt - baseline_dt).days + + rows = _iterate_date_range( + date_range=pd.date_range("2010-01-01", "2030-12-31", freq="D"), + additions=additions, + removals=removals, + planned_additions=planned_additions, + base_capacity_kw=base_capacity_kw, + target_gw=TARGET_SOLAR_GW, + baseline_dt=baseline_dt, + days_to_target=days_to_target, + heute=heute, + ) + return pd.DataFrame(rows) + + +def _iterate_date_range( # noqa: PLR0913 + date_range: pd.DatetimeIndex, + additions: pd.Series, + removals: pd.Series, + planned_additions: pd.Series, + base_capacity_kw: float, + target_gw: float, + baseline_dt: pd.Timestamp, + days_to_target: int, + heute: str, +) -> list[dict]: + """Iteriert über den Datumsbereich und berechnet Tagesdaten.""" + rows = [] + cumulative_kw = base_capacity_kw + cumulative_planned_gw = 0.0 + baseline_capacity_gw = None + cumulative_needed_gw = None + daily_needed_gw = None + + for day in date_range: + added_kw = additions.get(day, 0.0) + removed_kw = removals.get(day, 0.0) + net_kw = added_kw - removed_kw + cumulative_kw += net_kw + cumulative_gw = round(cumulative_kw / 1_000_000, 2) + daily_mw = round(net_kw / 1_000, 1) + + if day == baseline_dt: + baseline_capacity_gw = cumulative_gw + + noetig_gesamt = None + noetig_taeglich = None + if baseline_capacity_gw is not None and day >= baseline_dt: + if daily_needed_gw is None: + daily_needed_gw = (target_gw - baseline_capacity_gw) / days_to_target + cumulative_needed_gw = baseline_capacity_gw + cumulative_needed_gw += daily_needed_gw + noetig_gesamt = round(cumulative_needed_gw, 2) + noetig_taeglich = round(daily_needed_gw * 1000, 1) + + geplant_gesamt = None + geplant_taeglich = None + day_str = day.strftime("%Y-%m-%d") + if day_str >= heute: + planned_kw = planned_additions.get(day, 0.0) + cumulative_planned_gw += planned_kw / 1_000_000 + geplant_gesamt = round(cumulative_gw + cumulative_planned_gw, 2) + geplant_taeglich = round(planned_kw / 1_000, 1) + + installiert_gesamt = cumulative_gw if day_str <= heute else None + installiert_taeglich = daily_mw if day_str <= heute else None + + # Subtract existing from planned where both exist (avoid double-counting) + if installiert_gesamt is not None and geplant_gesamt is not None: + geplant_gesamt = round(geplant_gesamt - installiert_gesamt, 2) + if installiert_taeglich is not None and geplant_taeglich is not None: + geplant_taeglich = round(geplant_taeglich - installiert_taeglich, 1) + + row = { + "datum": day_str, + "installiert_gesamt": installiert_gesamt, + "installiert_taeglich": installiert_taeglich, + "geplant_gesamt": geplant_gesamt, + "geplant_taeglich": geplant_taeglich, + "noetig_gesamt": noetig_gesamt, + "noetig_taeglich": noetig_taeglich, + } + rows.append(row) + + return rows + + +def _aggregate_summaries(df: pd.DataFrame) -> dict[str, pd.DataFrame]: + """Erstellt monatliche und jährliche Zusammenfassungen für Solar.""" + df["_datum"] = pd.to_datetime(df["datum"]) + + results = {} + for freq, label in [("ME", "monatlich"), ("YE", "jaehrlich")]: + resampled = df.set_index("_datum").resample(freq) + + gesamt = pd.DataFrame({"datum": resampled.last().index}) + gesamt["datum"] = gesamt["datum"].dt.strftime("%Y-%m-%d") + last = resampled.last() + gesamt["installiert"] = last["installiert_gesamt"].values + gesamt["geplant"] = last["geplant_gesamt"].values + # Subtract existing from planned where both exist + mask = gesamt["installiert"].notna() & gesamt["geplant"].notna() + gesamt.loc[mask, "geplant"] -= gesamt.loc[mask, "installiert"] + results[f"gesamt_{label}"] = gesamt + + zubau = pd.DataFrame({"datum": resampled.sum(numeric_only=True, min_count=1).index}) + zubau["datum"] = zubau["datum"].dt.strftime("%Y-%m-%d") + summed = resampled.sum(numeric_only=True, min_count=1) + zubau["installiert"] = summed["installiert_taeglich"].values + zubau["geplant"] = summed["geplant_taeglich"].values + # Subtract existing from planned where both exist + mask = zubau["installiert"].notna() & zubau["geplant"].notna() + zubau.loc[mask, "geplant"] -= zubau.loc[mask, "installiert"] + results[f"zubau_{label}"] = zubau + + df.drop(columns=["_datum"], inplace=True) + return results + + +def process_solar(db_path: Path) -> tuple[pd.DataFrame, dict[str, pd.DataFrame]]: + """ + Berechnet die täglichen Ausbaudaten für Solarenergie. + + Liest aus solar_extended in mastr.db, schreibt ee_solar_taeglich. + Gibt (df_solar, summaries) zurück. + """ + db = sqlite3.connect(db_path) + _create_result_table(db) + + try: + df = pd.read_sql_query("SELECT * FROM solar_extended", db) + except Exception as e: + sentry_sdk.capture_exception(e) + msg = "solar_extended-Tabelle nicht gefunden in mastr.db" + raise RuntimeError(msg) from e + + print(f" {len(df)} Solar-Einheiten in der Datenbank.") + + print(" Berechne Solar-Ausbaudaten...") + df_solar = _calculate_daily_capacity(df) + + # In DB speichern + heute = datetime.now().strftime("%Y-%m-%d") + db.execute("DELETE FROM ee_solar_taeglich") + for _, row in df_solar.iterrows(): + db.execute( + """INSERT INTO ee_solar_taeglich + (datum, installiert_gesamt, installiert_taeglich, + geplant_gesamt, geplant_taeglich, noetig_gesamt, noetig_taeglich, stand) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + row["datum"], + row.get("installiert_gesamt"), + row.get("installiert_taeglich"), + row.get("geplant_gesamt"), + row.get("geplant_taeglich"), + row.get("noetig_gesamt"), + row.get("noetig_taeglich"), + heute, + ), + ) + db.commit() + db.close() + + print(f" {len(df_solar)} Tagesdatensätze berechnet.") + + print(" Berechne monatliche/jährliche Zusammenfassungen...") + summaries = _aggregate_summaries(df_solar) + + return df_solar, summaries + + +if __name__ == "__main__": + process_solar(Path(__file__).parent / "mastr.db") diff --git a/ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py b/ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py new file mode 100644 index 0000000..3bf10c4 --- /dev/null +++ b/ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py @@ -0,0 +1,321 @@ +""" +Wind-Prozessor: Berechnet tägliche Ausbaudaten (installiert, geplant, nötig) +aus der lokalen MaStR-Datenbank. + +Liest aus wind_extended, schreibt in ee_wind_taeglich. +Pendant zu msr_php/wka_to_data.php. +""" + +import sqlite3 +from datetime import datetime +from pathlib import Path + +import pandas as pd +import sentry_sdk + +# Ausbauziele 2030 +TARGET_ONSHORE_GW = 115 # Wind-an-Land-Gesetz, in Kraft seit 01.02.2023 +TARGET_OFFSHORE_GW = 30 # Wind-auf-See-Gesetz, in Kraft seit 01.01.2023 +TARGET_DATE = "2031-01-01" +BASELINE_ONSHORE = "2023-02-01" +BASELINE_OFFSHORE = "2023-01-01" + +# Lage-Filter (Werte aus open-mastr nach Bulk-Cleansing) +ONSHORE_LABELS = ("Windkraft an Land",) +OFFSHORE_LABELS = ("Windkraft auf See",) + +# Status-Filter für inaktive Anlagen +INACTIVE_STATUSES = ("In Planung", "Vorübergehend stillgelegt") + + +def _create_result_table(db: sqlite3.Connection): + """Erstellt die Ergebnis-Tabelle ee_wind_taeglich.""" + db.execute(""" + CREATE TABLE IF NOT EXISTS ee_wind_taeglich ( + datum TEXT NOT NULL, + lage_einheit TEXT NOT NULL, + installiert_gesamt REAL, + installiert_taeglich REAL, + geplant_gesamt REAL, + geplant_taeglich REAL, + noetig_gesamt REAL, + noetig_taeglich REAL, + stand TEXT NOT NULL, + PRIMARY KEY (datum, lage_einheit) + ) + """) + db.commit() + + +def _calculate_daily_capacity( + df: pd.DataFrame, + location_labels: tuple[str, ...], + target_gw: float, + baseline_date: str, +) -> pd.DataFrame: + """ + Berechnet für eine Lage (onshore/offshore) die täglichen Ausbaudaten. + + Identische Logik wie im Original msr_wind.py, angepasst an open-mastr-Spaltennamen. + """ + heute = datetime.now().strftime("%Y-%m-%d") + + # Aktive Anlagen filtern (nicht in Planung, nicht stillgelegt) + active = df[ + df["WindAnLandOderAufSee"].isin(location_labels) + & ~df["EinheitBetriebsstatus"].isin(INACTIVE_STATUSES) + ].copy() + active["Inbetriebnahmedatum"] = pd.to_datetime(active["Inbetriebnahmedatum"]) + active["datum_stilllegung"] = pd.to_datetime(active["datum_stilllegung"]) + active["Nettonennleistung"] = pd.to_numeric(active["Nettonennleistung"], errors="coerce") + + # Geplante Anlagen + planned = df[ + df["WindAnLandOderAufSee"].isin(location_labels) + & (df["EinheitBetriebsstatus"] == "In Planung") + ].copy() + planned["GeplantesInbetriebnahmedatum"] = pd.to_datetime( + planned["GeplantesInbetriebnahmedatum"] + ) + planned["Nettonennleistung"] = pd.to_numeric(planned["Nettonennleistung"], errors="coerce") + + # Vorleistung bis Ende 2009 + pre_2010 = active[active["Inbetriebnahmedatum"] < "2010-01-01"]["Nettonennleistung"].sum() + decom_pre_2010 = active[ + active["datum_stilllegung"].notna() & (active["datum_stilllegung"] < "2010-01-01") + ]["Nettonennleistung"].sum() + base_capacity_kw = pre_2010 - decom_pre_2010 + + # Zubau pro Tag (kW) + additions = ( + active[active["Inbetriebnahmedatum"] >= "2010-01-01"] + .groupby("Inbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + # Abbau pro Tag (kW) + removals = ( + active[active["datum_stilllegung"].notna() & (active["datum_stilllegung"] >= "2010-01-01")] + .groupby("datum_stilllegung")["Nettonennleistung"] + .sum() + ) + # Geplante Zubauten pro Tag (kW) + planned_additions = ( + planned[planned["GeplantesInbetriebnahmedatum"].notna()] + .groupby("GeplantesInbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + + baseline_dt = pd.Timestamp(baseline_date) + target_dt = pd.Timestamp(TARGET_DATE) + days_to_target = (target_dt - baseline_dt).days + + rows = _iterate_date_range( + date_range=pd.date_range("2010-01-01", "2030-12-31", freq="D"), + additions=additions, + removals=removals, + planned_additions=planned_additions, + base_capacity_kw=base_capacity_kw, + target_gw=target_gw, + baseline_dt=baseline_dt, + days_to_target=days_to_target, + heute=heute, + ) + return pd.DataFrame(rows) + + +def _iterate_date_range( # noqa: PLR0913 + date_range: pd.DatetimeIndex, + additions: pd.Series, + removals: pd.Series, + planned_additions: pd.Series, + base_capacity_kw: float, + target_gw: float, + baseline_dt: pd.Timestamp, + days_to_target: int, + heute: str, +) -> list[dict]: + """Iteriert über den Datumsbereich und berechnet Tagesdaten.""" + rows = [] + cumulative_kw = base_capacity_kw + cumulative_planned_gw = 0.0 + baseline_capacity_gw = None + cumulative_needed_gw = None + daily_needed_gw = None + + for day in date_range: + # Zubau/Abbau des Tages + added_kw = additions.get(day, 0.0) + removed_kw = removals.get(day, 0.0) + net_kw = added_kw - removed_kw + cumulative_kw += net_kw + cumulative_gw = round(cumulative_kw / 1_000_000, 2) + daily_mw = round(net_kw / 1_000, 1) + + # Baseline-Stand merken + if day == baseline_dt: + baseline_capacity_gw = cumulative_gw + + # Nötige Leistung (ab Baseline) + noetig_gesamt = None + noetig_taeglich = None + if baseline_capacity_gw is not None and day >= baseline_dt: + if daily_needed_gw is None: + daily_needed_gw = (target_gw - baseline_capacity_gw) / days_to_target + cumulative_needed_gw = baseline_capacity_gw + cumulative_needed_gw += daily_needed_gw + noetig_gesamt = round(cumulative_needed_gw, 2) + noetig_taeglich = round(daily_needed_gw * 1000, 1) + + # Geplante Zubauten (nur in Zukunft) + geplant_gesamt = None + geplant_taeglich = None + day_str = day.strftime("%Y-%m-%d") + if day_str >= heute: + planned_kw = planned_additions.get(day, 0.0) + cumulative_planned_gw += planned_kw / 1_000_000 + geplant_gesamt = round(cumulative_gw + cumulative_planned_gw, 2) + geplant_taeglich = round(planned_kw / 1_000, 1) + + installiert_gesamt = cumulative_gw if day_str <= heute else None + installiert_taeglich = daily_mw if day_str <= heute else None + + # Subtract existing from planned where both exist (avoid double-counting) + if installiert_gesamt is not None and geplant_gesamt is not None: + geplant_gesamt = round(geplant_gesamt - installiert_gesamt, 2) + if installiert_taeglich is not None and geplant_taeglich is not None: + geplant_taeglich = round(geplant_taeglich - installiert_taeglich, 1) + + row = { + "datum": day_str, + "installiert_gesamt": installiert_gesamt, + "installiert_taeglich": installiert_taeglich, + "geplant_gesamt": geplant_gesamt, + "geplant_taeglich": geplant_taeglich, + "noetig_gesamt": noetig_gesamt, + "noetig_taeglich": noetig_taeglich, + } + rows.append(row) + + return rows + + +def _aggregate_summaries( + df_onshore: pd.DataFrame, + df_offshore: pd.DataFrame, +) -> dict[str, pd.DataFrame]: + """Erstellt monatliche und jährliche Zusammenfassungen. + + Gibt dict mit 4 DataFrames zurück: + - gesamt_monatlich: Gesamtleistung (GW) zum Monatsende + - zubau_monatlich: Neue Kapazität (MW) pro Monat + - gesamt_jaehrlich: Gesamtleistung (GW) zum Jahresende + - zubau_jaehrlich: Neue Kapazität (MW) pro Jahr + """ + # Datum als Index für Resampling + for df in (df_onshore, df_offshore): + df["_datum"] = pd.to_datetime(df["datum"]) + + results = {} + for freq, label in [("ME", "monatlich"), ("YE", "jaehrlich")]: + # Gesamtleistung: letzter Wert der Periode + gesamt = pd.DataFrame({"datum": df_onshore.set_index("_datum").resample(freq).last().index}) + gesamt["datum"] = gesamt["datum"].dt.strftime("%Y-%m-%d") + + on = df_onshore.set_index("_datum").resample(freq).last() + off = df_offshore.set_index("_datum").resample(freq).last() + gesamt["onshore"] = on["installiert_gesamt"].values + gesamt["onshore_geplant"] = on["geplant_gesamt"].values + gesamt["offshore"] = off["installiert_gesamt"].values + gesamt["offshore_geplant"] = off["geplant_gesamt"].values + # Subtract existing from planned where both exist + for inst, plan in [("onshore", "onshore_geplant"), ("offshore", "offshore_geplant")]: + mask = gesamt[inst].notna() & gesamt[plan].notna() + gesamt.loc[mask, plan] -= gesamt.loc[mask, inst] + results[f"gesamt_{label}"] = gesamt + + # Zubau: Summe der Periode (MW) + zubau = pd.DataFrame({"datum": df_onshore.set_index("_datum").resample(freq).sum(numeric_only=True, min_count=1).index}) + zubau["datum"] = zubau["datum"].dt.strftime("%Y-%m-%d") + + on_sum = df_onshore.set_index("_datum").resample(freq).sum(numeric_only=True, min_count=1) + off_sum = df_offshore.set_index("_datum").resample(freq).sum(numeric_only=True, min_count=1) + zubau["onshore"] = on_sum["installiert_taeglich"].values + zubau["onshore_geplant"] = on_sum["geplant_taeglich"].values + zubau["offshore"] = off_sum["installiert_taeglich"].values + zubau["offshore_geplant"] = off_sum["geplant_taeglich"].values + # Subtract existing from planned where both exist + for inst, plan in [("onshore", "onshore_geplant"), ("offshore", "offshore_geplant")]: + mask = zubau[inst].notna() & zubau[plan].notna() + zubau.loc[mask, plan] -= zubau.loc[mask, inst] + results[f"zubau_{label}"] = zubau + + # Temporäre Spalte entfernen + for df in (df_onshore, df_offshore): + df.drop(columns=["_datum"], inplace=True) + + return results + + +def process_wind(db_path: Path) -> tuple[pd.DataFrame, pd.DataFrame, dict[str, pd.DataFrame]]: + """ + Berechnet die täglichen Ausbaudaten für Onshore und Offshore Wind. + + Liest aus wind_extended in mastr.db, schreibt ee_wind_taeglich. + Gibt (df_onshore, df_offshore, summaries) zurück. + """ + db = sqlite3.connect(db_path) + _create_result_table(db) + + try: + df = pd.read_sql_query("SELECT * FROM wind_extended", db) + except Exception as e: + sentry_sdk.capture_exception(e) + msg = "wind_extended-Tabelle nicht gefunden in mastr.db" + raise RuntimeError(msg) from e + + print(f" {len(df)} Wind-Einheiten in der Datenbank.") + + print(" Berechne Onshore-Daten...") + df_onshore = _calculate_daily_capacity(df, ONSHORE_LABELS, TARGET_ONSHORE_GW, BASELINE_ONSHORE) + df_onshore["lage_einheit"] = "Windkraft an Land" + + print(" Berechne Offshore-Daten...") + df_offshore = _calculate_daily_capacity(df, OFFSHORE_LABELS, TARGET_OFFSHORE_GW, BASELINE_OFFSHORE) + df_offshore["lage_einheit"] = "Windkraft auf See" + + # In DB speichern + heute = datetime.now().strftime("%Y-%m-%d") + db.execute("DELETE FROM ee_wind_taeglich") + for row_df in (df_onshore, df_offshore): + for _, row in row_df.iterrows(): + db.execute( + """INSERT INTO ee_wind_taeglich + (datum, lage_einheit, installiert_gesamt, installiert_taeglich, + geplant_gesamt, geplant_taeglich, noetig_gesamt, noetig_taeglich, stand) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + row["datum"], + row["lage_einheit"], + row.get("installiert_gesamt"), + row.get("installiert_taeglich"), + row.get("geplant_gesamt"), + row.get("geplant_taeglich"), + row.get("noetig_gesamt"), + row.get("noetig_taeglich"), + heute, + ), + ) + db.commit() + db.close() + + print(f" {len(df_onshore) + len(df_offshore)} Tagesdatensätze berechnet.") + + # Monatliche und jährliche Zusammenfassungen + print(" Berechne monatliche/jährliche Zusammenfassungen...") + summaries = _aggregate_summaries(df_onshore, df_offshore) + + return df_onshore, df_offshore, summaries + + +if __name__ == "__main__": + process_wind(Path(__file__).parent / "mastr.db") diff --git a/docs/superpowers/plans/2026-04-10-msr-open-mastr.md b/docs/superpowers/plans/2026-04-10-msr-open-mastr.md new file mode 100644 index 0000000..648a958 --- /dev/null +++ b/docs/superpowers/plans/2026-04-10-msr-open-mastr.md @@ -0,0 +1,749 @@ +# MaStR open-mastr Refactoring Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the SOAP-API-based `msr_wind.py` with two scripts (scraper + processor) using the `open-mastr` library, downloading all energy types into `mastr.db` with S3 integration. + +**Architecture:** A generic scraper (`msr_scraper.py`) downloads all MaStR data via open-mastr and writes it to a local `mastr.db`. A wind-specific processor (`msr_wind_processor.py`) reads from that DB and calculates daily expansion data. The orchestrator (`klimadashboard.py`) handles S3 download/upload of the DB file. + +**Tech Stack:** Python 3.11, open-mastr, SQLite, pandas, boto3 (via `ddj_cloud.utils.storage`) + +**Spec:** `docs/superpowers/specs/2026-04-02-msr-wind-open-mastr-design.md` + +--- + +## File Structure + +| File | Action | Responsibility | +|---|---|---| +| `pyproject.toml` | Modify | Add `open-mastr` dependency | +| `src/msr_scraper.py` | Create | Download all energy types from MaStR via open-mastr, write to `mastr.db` | +| `src/msr_wind_processor.py` | Create | Read wind data from `mastr.db`, calculate daily expansion stats, write `ee_wind_taeglich` | +| `klimadashboard.py` | Modify | Add S3 download/upload of `mastr.db`, call scraper + processor | +| `src/msr_wind.py` | Delete | Replaced by scraper + processor | +| `README_msr.md` | Modify | Update for new architecture | + +--- + +### Task 1: Add open-mastr dependency + +**Files:** +- Modify: `pyproject.toml:12-35` + +- [ ] **Step 1: Add open-mastr to dependencies** + +In `pyproject.toml`, add `open-mastr` to the `dependencies` list. Add after the `requests` line: + +```toml + "open-mastr~=0.17", +``` + +- [ ] **Step 2: Install dependencies** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv sync` +Expected: open-mastr and its dependencies install successfully. + +- [ ] **Step 3: Verify import works** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c "from open_mastr import Mastr; print('open-mastr OK')"` +Expected: `open-mastr OK` + +- [ ] **Step 4: Commit** + +```bash +git add pyproject.toml uv.lock +git commit -m "feat: add open-mastr dependency" +``` + +--- + +### Task 2: Create the MaStR scraper (`msr_scraper.py`) + +**Files:** +- Create: `src/msr_scraper.py` + +This scraper downloads all energy types from the MaStR bulk export via open-mastr and writes them into our own `mastr.db`. It copies relevant tables from the open-mastr DB to our DB, adding a computed `datum_stilllegung` column where applicable. + +- [ ] **Step 1: Create `src/msr_scraper.py`** + +```python +""" +MaStR-Scraper: Lädt alle Energiearten aus dem Marktstammdatenregister +über die open-mastr-Bibliothek (Bulk-Download, kein API-Key nötig). + +Schreibt die Daten in eine lokale SQLite-Datenbank (mastr.db). +""" + +import sqlite3 +from pathlib import Path + +import pandas as pd +import sentry_sdk +from open_mastr import Mastr + +# Energiearten, die heruntergeladen werden +ENERGY_TYPES = ["wind", "solar", "biomass", "hydro", "combustion", "nuclear", "gsgk", "storage"] + +# Mapping: open-mastr data-Name -> Tabellenname in der open-mastr-DB +TABLE_NAMES = { + "wind": "wind_extended", + "solar": "solar_extended", + "biomass": "biomass_extended", + "hydro": "hydro_extended", + "combustion": "combustion_extended", + "nuclear": "nuclear_extended", + "gsgk": "gsgk_extended", + "storage": "storage_extended", +} + +# Technologien mit Stilllegungsdaten (max aus zwei Datumsfeldern) +TECHS_WITH_SHUTDOWN = {"wind", "solar", "biomass", "hydro", "combustion", "nuclear", "gsgk", "storage"} + + +def _compute_shutdown_date(df: pd.DataFrame) -> pd.DataFrame: + """Berechnet datum_stilllegung als max(DatumBeginnVoruebergehendeStilllegung, DatumEndgueltigeStilllegung).""" + col_temp = "DatumBeginnVoruebergehendeStilllegung" + col_final = "DatumEndgueltigeStilllegung" + + if col_temp not in df.columns and col_final not in df.columns: + return df + + if col_temp in df.columns and col_final in df.columns: + temp = pd.to_datetime(df[col_temp], errors="coerce") + final = pd.to_datetime(df[col_final], errors="coerce") + df["datum_stilllegung"] = temp.where(temp > final, final) + df["datum_stilllegung"] = df["datum_stilllegung"].dt.strftime("%Y-%m-%d") + elif col_temp in df.columns: + df["datum_stilllegung"] = df[col_temp] + else: + df["datum_stilllegung"] = df[col_final] + + return df + + +def _get_open_mastr_db_path() -> Path: + """Gibt den Pfad zur open-mastr SQLite-Datenbank zurück.""" + return Path.home() / ".open-MaStR" / "data" / "sqlite" / "open-mastr.db" + + +def scrape_mastr(db_path: Path) -> dict[str, int]: + """ + Lädt alle Energiearten via open-mastr und schreibt sie in die lokale DB. + + Args: + db_path: Pfad zur Ziel-SQLite-Datenbank (mastr.db) + + Returns: + Dict mit Anzahl der Einheiten pro Energieart. + """ + print("MaStR Bulk-Download starten...") + + # Schritt 1: open-mastr Bulk-Download + try: + mastr = Mastr() + mastr.download(data=ENERGY_TYPES) + except Exception as e: + sentry_sdk.capture_exception(e) + raise RuntimeError("open-mastr Bulk-Download fehlgeschlagen") from e + + # Schritt 2: Daten aus open-mastr-DB lesen und in unsere DB schreiben + open_mastr_db = _get_open_mastr_db_path() + if not open_mastr_db.exists(): + msg = f"open-mastr-DB nicht gefunden: {open_mastr_db}" + raise FileNotFoundError(msg) + + counts = {} + source_conn = sqlite3.connect(open_mastr_db) + target_conn = sqlite3.connect(db_path) + + try: + for energy_type, table_name in TABLE_NAMES.items(): + print(f" Kopiere {energy_type} ({table_name})...") + try: + df = pd.read_sql_query(f"SELECT * FROM {table_name}", source_conn) # noqa: S608 + except Exception as e: + print(f" Warnung: Tabelle {table_name} nicht gefunden: {e}") + sentry_sdk.capture_exception(e) + continue + + if energy_type in TECHS_WITH_SHUTDOWN: + df = _compute_shutdown_date(df) + + # In unsere DB schreiben (Tabelle komplett ersetzen) + df.to_sql(table_name, target_conn, if_exists="replace", index=False) + counts[energy_type] = len(df) + print(f" {len(df)} Einheiten für {energy_type} geschrieben.") + + target_conn.commit() + finally: + source_conn.close() + target_conn.close() + + print(f"MaStR-Daten in {db_path} geschrieben: {counts}") + return counts + + +if __name__ == "__main__": + scrape_mastr(Path(__file__).parent / "mastr.db") +``` + +- [ ] **Step 2: Verify syntax** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c "import ast; ast.parse(open('ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py').read()); print('Syntax OK')"` +Expected: `Syntax OK` + +- [ ] **Step 3: Run linter** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run ruff check ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py` +Expected: No errors (fix any that appear). + +- [ ] **Step 4: Commit** + +```bash +git add ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py +git commit -m "feat: add MaStR scraper using open-mastr bulk download" +``` + +--- + +### Task 3: Create the wind processor (`msr_wind_processor.py`) + +**Files:** +- Create: `src/msr_wind_processor.py` + +This processor reads wind units from `mastr.db` (table `wind_extended`) and calculates daily expansion data (installed, planned, needed for 2030 targets). The calculation logic is ported from the existing `_calculate_daily_capacity()` in `msr_wind.py`. + +Key differences from `msr_wind.py`: +- Reads from `wind_extended` table (open-mastr schema) instead of `ee_wind` +- Column names: `Lage` (not `lage_einheit`), `EinheitBetriebsstatus` (not `betriebsstatus`), `Nettonennleistung` (not `nettonennleistung`), `Inbetriebnahmedatum` (not `datum_inbetriebnahme`), `GeplantesInbetriebnahmedatum` (not `datum_geplante_inbetriebnahme`), `datum_stilllegung` (computed by scraper) +- Status values after bulk cleansing: `"In Planung"`, `"Vorübergehend stillgelegt"` (space-separated German, not CamelCase) + +- [ ] **Step 1: Create `src/msr_wind_processor.py`** + +```python +""" +Wind-Prozessor: Berechnet tägliche Ausbaudaten (installiert, geplant, nötig) +aus der lokalen MaStR-Datenbank. + +Liest aus wind_extended, schreibt in ee_wind_taeglich. +Pendant zu msr_php/wka_to_data.php. +""" + +import sqlite3 +from datetime import datetime +from pathlib import Path + +import pandas as pd +import sentry_sdk + +# Ausbauziele 2030 +TARGET_ONSHORE_GW = 115 # Wind-an-Land-Gesetz, in Kraft seit 01.02.2023 +TARGET_OFFSHORE_GW = 30 # Wind-auf-See-Gesetz, in Kraft seit 01.01.2023 +TARGET_DATE = "2031-01-01" +BASELINE_ONSHORE = "2023-02-01" +BASELINE_OFFSHORE = "2023-01-01" + +# Lage-Filter (Werte aus open-mastr nach Bulk-Cleansing) +ONSHORE_LABELS = ("Windkraft an Land",) +OFFSHORE_LABELS = ("Windkraft auf See",) + +# Status-Filter für inaktive Anlagen +INACTIVE_STATUSES = ("In Planung", "Vorübergehend stillgelegt") + + +def _create_result_table(db: sqlite3.Connection): + """Erstellt die Ergebnis-Tabelle ee_wind_taeglich.""" + db.execute(""" + CREATE TABLE IF NOT EXISTS ee_wind_taeglich ( + datum TEXT NOT NULL, + lage_einheit TEXT NOT NULL, + installiert_gesamt REAL, + installiert_taeglich REAL, + geplant_gesamt REAL, + geplant_taeglich REAL, + noetig_gesamt REAL, + noetig_taeglich REAL, + stand TEXT NOT NULL, + PRIMARY KEY (datum, lage_einheit) + ) + """) + db.commit() + + +def _calculate_daily_capacity( + df: pd.DataFrame, + location_labels: tuple[str, ...], + target_gw: float, + baseline_date: str, +) -> pd.DataFrame: + """ + Berechnet für eine Lage (onshore/offshore) die täglichen Ausbaudaten. + + Identische Logik wie im Original msr_wind.py, angepasst an open-mastr-Spaltennamen. + """ + heute = datetime.now().strftime("%Y-%m-%d") + + # Aktive Anlagen filtern (nicht in Planung, nicht stillgelegt) + active = df[ + df["Lage"].isin(location_labels) + & ~df["EinheitBetriebsstatus"].isin(INACTIVE_STATUSES) + ].copy() + active["Inbetriebnahmedatum"] = pd.to_datetime(active["Inbetriebnahmedatum"]) + active["datum_stilllegung"] = pd.to_datetime(active["datum_stilllegung"]) + active["Nettonennleistung"] = pd.to_numeric(active["Nettonennleistung"], errors="coerce") + + # Geplante Anlagen + planned = df[ + df["Lage"].isin(location_labels) + & (df["EinheitBetriebsstatus"] == "In Planung") + ].copy() + planned["GeplantesInbetriebnahmedatum"] = pd.to_datetime( + planned["GeplantesInbetriebnahmedatum"] + ) + planned["Nettonennleistung"] = pd.to_numeric(planned["Nettonennleistung"], errors="coerce") + + # Vorleistung bis Ende 2009 + pre_2010 = active[active["Inbetriebnahmedatum"] < "2010-01-01"]["Nettonennleistung"].sum() + decom_pre_2010 = active[ + active["datum_stilllegung"].notna() & (active["datum_stilllegung"] < "2010-01-01") + ]["Nettonennleistung"].sum() + base_capacity_kw = pre_2010 - decom_pre_2010 + + # Tägliche Zubau-/Abbau-Summen berechnen + date_range = pd.date_range("2010-01-01", "2030-12-31", freq="D") + + # Zubau pro Tag (kW) + additions = ( + active[active["Inbetriebnahmedatum"] >= "2010-01-01"] + .groupby("Inbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + # Abbau pro Tag (kW) + removals = ( + active[active["datum_stilllegung"].notna() & (active["datum_stilllegung"] >= "2010-01-01")] + .groupby("datum_stilllegung")["Nettonennleistung"] + .sum() + ) + # Geplante Zubauten pro Tag (kW) + planned_additions = ( + planned[planned["GeplantesInbetriebnahmedatum"].notna()] + .groupby("GeplantesInbetriebnahmedatum")["Nettonennleistung"] + .sum() + ) + + rows = [] + cumulative_kw = base_capacity_kw + cumulative_planned_gw = 0.0 + + # Nötige Rate berechnen + baseline_dt = pd.Timestamp(baseline_date) + target_dt = pd.Timestamp(TARGET_DATE) + days_to_target = (target_dt - baseline_dt).days + baseline_capacity_gw = None + cumulative_needed_gw = None + daily_needed_gw = None + + for day in date_range: + # Zubau/Abbau des Tages + added_kw = additions.get(day, 0.0) + removed_kw = removals.get(day, 0.0) + net_kw = added_kw - removed_kw + cumulative_kw += net_kw + cumulative_gw = round(cumulative_kw / 1_000_000, 2) + daily_mw = round(net_kw / 1_000, 1) + + # Baseline-Stand merken + if day == baseline_dt: + baseline_capacity_gw = cumulative_gw + + # Nötige Leistung (ab Baseline) + noetig_gesamt = None + noetig_taeglich = None + if baseline_capacity_gw is not None and day >= baseline_dt: + if daily_needed_gw is None: + daily_needed_gw = (target_gw - baseline_capacity_gw) / days_to_target + cumulative_needed_gw = baseline_capacity_gw + cumulative_needed_gw += daily_needed_gw + noetig_gesamt = round(cumulative_needed_gw, 2) + noetig_taeglich = round(daily_needed_gw * 1000, 1) + + # Geplante Zubauten (nur in Zukunft) + geplant_gesamt = None + geplant_taeglich = None + day_str = day.strftime("%Y-%m-%d") + if day_str >= heute: + planned_kw = planned_additions.get(day, 0.0) + cumulative_planned_gw += planned_kw / 1_000_000 + geplant_gesamt = round(cumulative_gw + cumulative_planned_gw, 2) + geplant_taeglich = round(planned_kw / 1_000, 1) + + row = { + "datum": day_str, + "installiert_gesamt": cumulative_gw if day_str <= heute else None, + "installiert_taeglich": daily_mw if day_str <= heute else None, + "geplant_gesamt": geplant_gesamt, + "geplant_taeglich": geplant_taeglich, + "noetig_gesamt": noetig_gesamt, + "noetig_taeglich": noetig_taeglich, + } + rows.append(row) + + return pd.DataFrame(rows) + + +def process_wind(db_path: Path) -> tuple[pd.DataFrame, pd.DataFrame]: + """ + Berechnet die täglichen Ausbaudaten für Onshore und Offshore Wind. + + Liest aus wind_extended in mastr.db, schreibt ee_wind_taeglich. + Gibt (df_onshore, df_offshore) zurück. + """ + db = sqlite3.connect(db_path) + _create_result_table(db) + + try: + df = pd.read_sql_query("SELECT * FROM wind_extended", db) + except Exception as e: + sentry_sdk.capture_exception(e) + raise RuntimeError("wind_extended-Tabelle nicht gefunden in mastr.db") from e + + print(f" {len(df)} Wind-Einheiten in der Datenbank.") + + print(" Berechne Onshore-Daten...") + df_onshore = _calculate_daily_capacity(df, ONSHORE_LABELS, TARGET_ONSHORE_GW, BASELINE_ONSHORE) + df_onshore["lage_einheit"] = "Windkraft an Land" + + print(" Berechne Offshore-Daten...") + df_offshore = _calculate_daily_capacity(df, OFFSHORE_LABELS, TARGET_OFFSHORE_GW, BASELINE_OFFSHORE) + df_offshore["lage_einheit"] = "Windkraft auf See" + + # In DB speichern + heute = datetime.now().strftime("%Y-%m-%d") + db.execute("DELETE FROM ee_wind_taeglich") + for row_df in (df_onshore, df_offshore): + for _, row in row_df.iterrows(): + db.execute( + """INSERT INTO ee_wind_taeglich + (datum, lage_einheit, installiert_gesamt, installiert_taeglich, + geplant_gesamt, geplant_taeglich, noetig_gesamt, noetig_taeglich, stand) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + row["datum"], + row["lage_einheit"], + row.get("installiert_gesamt"), + row.get("installiert_taeglich"), + row.get("geplant_gesamt"), + row.get("geplant_taeglich"), + row.get("noetig_gesamt"), + row.get("noetig_taeglich"), + heute, + ), + ) + db.commit() + db.close() + + print(f" {len(df_onshore) + len(df_offshore)} Tagesdatensätze berechnet.") + return df_onshore, df_offshore + + +if __name__ == "__main__": + process_wind(Path(__file__).parent / "mastr.db") +``` + +- [ ] **Step 2: Verify syntax** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c "import ast; ast.parse(open('ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py').read()); print('Syntax OK')"` +Expected: `Syntax OK` + +- [ ] **Step 3: Run linter** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run ruff check ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py` +Expected: No errors (fix any that appear). + +- [ ] **Step 4: Commit** + +```bash +git add ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py +git commit -m "feat: add wind processor with daily expansion calculation" +``` + +--- + +### Task 4: Update klimadashboard.py with S3 integration + +**Files:** +- Modify: `klimadashboard.py` + +Add S3 download/upload of `mastr.db` and call the new scraper + processor. + +- [ ] **Step 1: Update `klimadashboard.py`** + +Replace the entire file content with: + +```python +from pathlib import Path + +from ddj_cloud.scrapers.klimadashboard.src.energiemix import update_energiemix +from ddj_cloud.scrapers.klimadashboard.src.msr_scraper import scrape_mastr +from ddj_cloud.scrapers.klimadashboard.src.msr_wind_processor import process_wind +from ddj_cloud.utils.storage import ( + DownloadFailedException, + download_file, + upload_dataframe, + upload_file, +) + +VERSION_STRING = "V0.02 vom 10.04.2026" + +DB_S3_KEY = "klimadashboard/mastr.db" +DB_LOCAL_PATH = Path(__file__).parent / "src" / "mastr.db" + + +def _download_db(): + """Lädt mastr.db von S3 herunter (falls vorhanden).""" + try: + bio = download_file(DB_S3_KEY) + DB_LOCAL_PATH.write_bytes(bio.read()) + print(f" mastr.db von S3 heruntergeladen ({DB_LOCAL_PATH.stat().st_size / 1024 / 1024:.1f} MB)") + except DownloadFailedException: + print(" Keine mastr.db auf S3 gefunden (erster Lauf).") + + +def _upload_db(): + """Lädt mastr.db auf S3 hoch.""" + if not DB_LOCAL_PATH.exists(): + print(" Warnung: mastr.db nicht gefunden, Upload übersprungen.") + return + upload_file( + DB_LOCAL_PATH.read_bytes(), + DB_S3_KEY, + archive=False, + ) + print(f" mastr.db auf S3 hochgeladen ({DB_LOCAL_PATH.stat().st_size / 1024 / 1024:.1f} MB)") + + +def run(): + # Energiemix (Fraunhofer API) + df = update_energiemix() + upload_dataframe(df, "klimadashboard/test_energiemix1.csv") + + # MaStR Wind-Ausbau + print("MaStR Wind-Daten aktualisieren...") + _download_db() + scrape_mastr(DB_LOCAL_PATH) + df_onshore, df_offshore = process_wind(DB_LOCAL_PATH) + _upload_db() + print("MaStR Wind-Daten aktualisiert.") +``` + +- [ ] **Step 2: Run linter** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run ruff check ddj_cloud/scrapers/klimadashboard/klimadashboard.py` +Expected: No errors. + +- [ ] **Step 3: Commit** + +```bash +git add ddj_cloud/scrapers/klimadashboard/klimadashboard.py +git commit -m "feat: integrate MaStR scraper+processor with S3 download/upload" +``` + +--- + +### Task 5: Delete old msr_wind.py and update docs + +**Files:** +- Delete: `src/msr_wind.py` +- Modify: `README_msr.md` +- Modify: `README.md` + +- [ ] **Step 1: Delete `src/msr_wind.py`** + +Run: `rm ddj_cloud/scrapers/klimadashboard/src/msr_wind.py` + +- [ ] **Step 2: Update `README_msr.md`** + +Replace entire content with: + +```markdown +# MaStR-Scraper und Wind-Prozessor + +Python-Port der PHP-Skripte `msr_php/wka_daily.php` und `msr_php/wka_to_data.php`, +jetzt basierend auf der [open-mastr](https://github.com/OpenEnergyPlatform/open-mastr)-Bibliothek. + +## Architektur + +``` +klimadashboard.py (Orchestrator) + │ + ├── S3: download mastr.db + ├── msr_scraper.py → alle Energiearten aus MaStR + ├── msr_wind_processor.py → Wind-Tagesdaten berechnen + ├── S3: upload mastr.db + └── Datawrapper-Upload +``` + +### 1. Scraper (`src/msr_scraper.py`) + +Lädt alle Energiearten (Wind, Solar, Biomasse, Wasser, Kernkraft, Verbrennung, Geothermie/Grubengas, Speicher) +über den open-mastr Bulk-Download und speichert sie in `mastr.db`. + +**Kein API-Key nötig** -- nutzt die öffentlichen Bulk-Daten des MaStR. + +### 2. Wind-Prozessor (`src/msr_wind_processor.py`) + +Berechnet tägliche Ausbaudaten (2010-2030) für Onshore und Offshore Wind: +- Kumulierte installierte Leistung (GW) +- Täglicher Zubau/Abbau (MW) +- Geplante zukünftige Installationen +- Nötiger täglicher Ausbau für die Klimaschutzziele 2030 + +**Klimaziele 2030:** +- Onshore: 115 GW (Wind-an-Land-Gesetz, seit 01.02.2023) +- Offshore: 30 GW (Wind-auf-See-Gesetz, seit 01.01.2023) + +## Benötigte Secrets / Umgebungsvariablen + +| Variable | Beschreibung | Wo beantragen? | +|----------|-------------|----------------| +| `DATAWRAPPER_API_KEY` | API-Token für Datawrapper-Charts | [Datawrapper Account Settings](https://app.datawrapper.de/account/api-tokens) | +| `BUCKET_NAME` | S3-Bucket für mastr.db | AWS-Konfiguration | + +**Nicht mehr nötig:** `MASTR_API_KEY`, `MASTR_AKTEUR_NR` (open-mastr nutzt öffentliche Bulk-Daten). + +## Datenbank + +Die SQLite-Datenbank `mastr.db` wird auf S3 gespeichert und bei jedem Lauf heruntergeladen/hochgeladen. + +**Tabellen aus MaStR** (open-mastr-Schema): +- `wind_extended`, `solar_extended`, `biomass_extended`, `hydro_extended`, + `combustion_extended`, `nuclear_extended`, `gsgk_extended`, `storage_extended` + +**Berechnete Tabellen:** +- `ee_wind_taeglich`: Tägliche Ausbaudaten (installiert, geplant, nötig) pro Lage (onshore/offshore) + +## Unterschiede zum PHP-Original + +| Aspekt | PHP | Python | +|--------|-----|--------| +| Datenquelle | MaStR SOAP-API (API-Key nötig) | open-mastr Bulk-Download (kein Key) | +| Datenbank | MySQL (remote) | SQLite auf S3 | +| Datenverarbeitung | SQL-Queries pro Tag | pandas (vektorisiert) | +| Energiearten | Nur Wind | Alle (Wind, Solar, Biomasse, etc.) | +| Architektur | wka_daily.php + wka_to_data.php | msr_scraper.py + msr_wind_processor.py | + +## Erweiterbarkeit + +Weitere Prozessoren können hinzugefügt werden, die auf denselben Daten in `mastr.db` arbeiten: +- `msr_solar_processor.py` (Ausbauziel: 215 GW, EEG 2023) +- `msr_biomasse_processor.py` +``` + +- [ ] **Step 3: Update `README.md`** + +In `README.md`, replace the "Wind-Ausbau" section (lines 18-25) with: + +```markdown +### MaStR-Daten / Wind-Ausbau (`src/msr_scraper.py` + `src/msr_wind_processor.py`) + +Holt alle Energiearten aus dem Marktstammdatenregister über [open-mastr](https://github.com/OpenEnergyPlatform/open-mastr) (Bulk-Download, kein API-Key nötig) und berechnet tägliche Wind-Ausbaudaten (installiert, geplant, nötig für Klimaziel 2030). + +- Onshore-Ziel: 115 GW bis 2030 (Wind-an-Land-Gesetz) +- Offshore-Ziel: 30 GW bis 2030 (Wind-auf-See-Gesetz) + +**Benötigte Umgebungsvariablen:** `DATAWRAPPER_API_KEY` + +Siehe [README_msr.md](README_msr.md) für Details zur Architektur. +``` + +- [ ] **Step 4: Commit** + +```bash +git add -A ddj_cloud/scrapers/klimadashboard/ +git commit -m "refactor: remove old msr_wind.py, update documentation" +``` + +--- + +### Task 6: Add mastr.db to .gitignore + +**Files:** +- Modify or create: `.gitignore` (in klimadashboard directory or project root) + +The SQLite database should not be committed to git. + +- [ ] **Step 1: Check existing .gitignore** + +Run: `cat /Users/janeggers/Code/wdr-ddj-cloud/.gitignore 2>/dev/null || echo "no .gitignore"` + +- [ ] **Step 2: Add mastr.db and open-mastr cache to .gitignore** + +Add to the project's `.gitignore` (create `ddj_cloud/scrapers/klimadashboard/.gitignore` if a local one is preferred): + +``` +# MaStR databases +*.db +``` + +- [ ] **Step 3: Commit** + +```bash +git add ddj_cloud/scrapers/klimadashboard/.gitignore +git commit -m "chore: add .gitignore for MaStR database files" +``` + +--- + +### Task 7: Verify end-to-end (manual smoke test) + +- [ ] **Step 1: Run the scraper standalone** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python ddj_cloud/scrapers/klimadashboard/src/msr_scraper.py` + +Expected: Downloads MaStR bulk data, creates `src/mastr.db` with all energy type tables. This will take a few minutes on first run. + +- [ ] **Step 2: Verify database contents** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c " +import sqlite3 +db = sqlite3.connect('ddj_cloud/scrapers/klimadashboard/src/mastr.db') +tables = [r[0] for r in db.execute(\"SELECT name FROM sqlite_master WHERE type='table'\").fetchall()] +print('Tables:', tables) +for t in tables: + count = db.execute(f'SELECT COUNT(*) FROM {t}').fetchone()[0] + print(f' {t}: {count} rows') +db.close() +"` + +Expected: Tables for wind_extended, solar_extended, etc. with thousands of rows each. + +- [ ] **Step 3: Check wind Lage values** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c " +import sqlite3 +db = sqlite3.connect('ddj_cloud/scrapers/klimadashboard/src/mastr.db') +print('Lage values:', db.execute('SELECT DISTINCT Lage FROM wind_extended').fetchall()) +print('Status values:', db.execute('SELECT DISTINCT EinheitBetriebsstatus FROM wind_extended').fetchall()) +db.close() +"` + +Expected: Lage should include "Windkraft an Land" and "Windkraft auf See". If values differ, update `ONSHORE_LABELS` and `OFFSHORE_LABELS` in `msr_wind_processor.py`. + +- [ ] **Step 4: Run the processor standalone** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python ddj_cloud/scrapers/klimadashboard/src/msr_wind_processor.py` + +Expected: Reads wind data, calculates daily stats, prints count of records. + +- [ ] **Step 5: Verify results** + +Run: `cd /Users/janeggers/Code/wdr-ddj-cloud && uv run python -c " +import sqlite3 +db = sqlite3.connect('ddj_cloud/scrapers/klimadashboard/src/mastr.db') +rows = db.execute('SELECT * FROM ee_wind_taeglich WHERE datum = \"2025-01-01\"').fetchall() +for r in rows: + print(r) +db.close() +"` + +Expected: Two rows (onshore + offshore) with plausible GW values for 2025-01-01. Onshore should be ~62-65 GW, offshore ~8-9 GW. \ No newline at end of file diff --git a/docs/superpowers/specs/2026-04-02-msr-wind-open-mastr-design.md b/docs/superpowers/specs/2026-04-02-msr-wind-open-mastr-design.md new file mode 100644 index 0000000..e33128f --- /dev/null +++ b/docs/superpowers/specs/2026-04-02-msr-wind-open-mastr-design.md @@ -0,0 +1,135 @@ +# Design: MaStR-Daten mit open-mastr und Scraper/Prozessor-Trennung + +## Zusammenfassung + +Umbau von `src/msr_wind.py` auf die `open-mastr`-Bibliothek (kein API-Key mehr nötig) und Aufteilung in zwei Skripte analog zum PHP-Original (`wka_daily.php` + `wka_to_data.php`). Integration des S3-Download/Upload-Musters aus dem Talsperren-Scraper. Alle Energiearten werden heruntergeladen und in `mastr.db` gespeichert. + +## Architektur + +``` +klimadashboard.py (Orchestrator) + │ + ├── S3: download mastr.db + ├── msr_scraper.py → befüllt alle Energiearten-Tabellen in mastr.db + ├── msr_wind_processor.py → berechnet ee_wind_taeglich aus wind-Daten + ├── S3: upload mastr.db + └── Datawrapper-Upload (bestehend) +``` + +## Dateien + +| Datei | Rolle | Analog PHP | +|---|---|---| +| `src/msr_scraper.py` | Alle Energiearten aus MaStR holen via open-mastr | `wka_daily.php` | +| `src/msr_wind_processor.py` | Wind-Tagesdaten berechnen aus lokaler DB | `wka_to_data.php` | +| `klimadashboard.py` | Orchestrierung, S3-Integration | -- | +| `src/msr_wind.py` | Wird entfernt (ersetzt durch die zwei neuen) | -- | + +## Scraper (`msr_scraper.py`) + +### Ablauf + +1. `open_mastr.Mastr()` initialisieren (Standard-DB unter `~/.open-MaStR/`) +2. `db.download()` -- lädt Bulk-XML-Dumps aller Energiearten vom MaStR-Portal, kein API-Key nötig +3. Einheiten aus allen `*_extended`-Tabellen der open-mastr-DB lesen (via pandas/sqlite3) +4. Stilllegungsdatum berechnen: `max(DatumBeginnVoruebergehendeStilllegung, DatumEndgueltigeStilllegung)` als abgeleitete Spalte (wo vorhanden) +5. Per `INSERT OR REPLACE` in unsere `mastr.db` mergen -- eine Tabelle pro Energieart +6. Spaltennamen werden 1:1 aus open-mastr übernommen (kein Mapping) + +### Energiearten + +Alle von open-mastr unterstützten Technologien: +- Wind (`wind_extended`) +- Solar (`solar_extended`) +- Biomasse (`biomass_extended`) +- Wasser (`hydro_extended`) +- Kernkraft (`nuclear_extended`) +- Speicher (`storage_extended`) +- Geothermie, Verbrennung, etc. + +### Hauptfunktion + +```python +def scrape_mastr(db_path: Path) -> dict[str, int]: + """Lädt alle Energiearten via open-mastr und schreibt sie in die lokale DB. + Gibt dict mit Anzahl der Einheiten pro Energieart zurück.""" +``` + +## Prozessor (`msr_wind_processor.py`) + +### Ablauf + +1. Wind-Einheiten aus `wind_extended`-Tabelle in `mastr.db` lesen +2. Für Onshore und Offshore jeweils: + - Vorleistung bis Ende 2009 berechnen + - Täglichen Zubau/Abbau 2010-2030 (kumuliert) + - Geplante Installationen (nur Zukunft) + - Nötigen täglichen Ausbau für Klimaziel 2030 +3. Ergebnis in `ee_wind_taeglich`-Tabelle schreiben + +### Klimaziele + +- Onshore: 115 GW bis 2031-01-01 (Wind-an-Land-Gesetz, Baseline 2023-02-01) +- Offshore: 30 GW bis 2031-01-01 (Wind-auf-See-Gesetz, Baseline 2023-01-01) + +### Anpassungen gegenüber aktuellem Code + +- Filter für Lage/Status: Werte aus open-mastr-DB verwenden (bei Implementierung verifizieren) +- `ee_wind_taeglich`-Schema bleibt gleich (eigene Berechnung) +- Berechnungslogik bleibt identisch zu `_calculate_daily_capacity()` + +### Hauptfunktion + +```python +def process_wind(db_path: Path) -> tuple[pd.DataFrame, pd.DataFrame]: + """Berechnet tägliche Ausbaudaten aus der lokalen DB. + Gibt (df_onshore, df_offshore) zurück.""" +``` + +## S3-Integration (`klimadashboard.py`) + +### Download vor Verarbeitung + +```python +from ddj_cloud.utils.storage import download_file, upload_file, DownloadFailedException + +DB_S3_KEY = "klimadashboard/mastr.db" +DB_LOCAL_PATH = Path(__file__).parent / "src" / "mastr.db" + +try: + bio = download_file(DB_S3_KEY) + DB_LOCAL_PATH.write_bytes(bio.read()) +except DownloadFailedException: + pass # Erster Lauf -- DB wird vom Scraper erstellt +``` + +### Upload nach Verarbeitung + +```python +upload_file(DB_LOCAL_PATH.read_bytes(), DB_S3_KEY, archive=False) +``` + +`archive=False` da die DB bei jedem Lauf mehrere MB groß ist. + +## Dependencies + +- `open-mastr` (PyPI: `open-mastr`, aktuell v0.16.1) -- neue Dependency +- Kein `MASTR_API_KEY` und `MASTR_AKTEUR_NR` mehr nötig +- `DATAWRAPPER_API_KEY` weiterhin nötig (für Chart-Upload) + +## Error Handling + +- Sentry-Integration wie bisher: `sentry_sdk.capture_exception(e)` bei Fehlern +- open-mastr-Download-Fehler: Scraper bricht ab, Prozessor läuft nicht +- S3-Download-Fehler beim ersten Lauf: OK, Scraper erstellt neue DB + +## Migration + +- `src/msr_wind.py` wird gelöscht +- `README_msr.md` wird aktualisiert (keine API-Keys mehr, neue Architektur) +- `klimadashboard.py` wird erweitert um Wind-Aufruf mit S3-Handling +- `MASTR_API_KEY` und `MASTR_AKTEUR_NR` können aus der Konfiguration entfernt werden + +## Erweiterbarkeit + +Die `mastr.db` enthält alle Energiearten. Weitere Prozessoren (z.B. `msr_solar_processor.py`, `msr_biomasse_processor.py`) können später hinzugefügt werden und arbeiten auf denselben Daten -- ohne erneuten Download. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index be863ba..d37d29c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "pydantic~=2.9.2", "shapely~=2.1", "requests~=2.32", + "open-mastr~=0.17", ] [project.scripts] diff --git a/scrapers_config.json b/scrapers_config.json index 2bd2bb6..985fcf5 100644 --- a/scrapers_config.json +++ b/scrapers_config.json @@ -259,5 +259,27 @@ ], "extra_env": [], "deploy": true + }, + { + "display_name": "klimadashboard", + "module_name": "klimadashboard", + "description": "Automation für Quarks.de: Ausbau von Wind- und Solarenergie, Energiemix in D und mehr\n", + "contact_name": "Jan Eggers", + "contact_email": "jan.eggers@fm.wdr.de", + "memory_size": "1024", + "ephemeral_storage": "512", + "preset": "pandas", + "events": [ + { + "type": "schedule", + "enabled": true, + "data": { + "interval": "daily", + "interval_custom": null + } + } + ], + "extra_env": ["DW_API_KEY_JE"], + "deploy": true } ] \ No newline at end of file diff --git a/uv.lock b/uv.lock index 0b00302..56d27d6 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "awscli" version = "1.44.59" @@ -517,6 +526,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/69/28/23eea8acd65972bbfe295ce3666b28ac510dfcb115fac089d3edb0feb00a/googleapis_common_protos-1.73.0-py3-none-any.whl", hash = "sha256:dfdaaa2e860f242046be561e6d6cb5c5f1541ae02cfbcb034371aadb2942b4e8", size = 297578, upload-time = "2026-03-06T21:52:33.933Z" }, ] +[[package]] +name = "greenlet" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, + { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, + { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, +] + [[package]] name = "grpcio" version = "1.78.0" @@ -618,6 +643,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d9/33/1f075bf72b0b747cb3288d011319aaf64083cf2efef8354174e3ed4540e2/ipython_pygments_lexers-1.1.1-py3-none-any.whl", hash = "sha256:a9462224a505ade19a605f71f8fa63c2048833ce50abc86768a0d81d876dc81c", size = 8074, upload-time = "2025-01-17T11:24:33.271Z" }, ] +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/27/056e0638a86749374d6f57d0b0db39f29509cce9313cf91bdc0ac4d91084/jaraco_functools-4.4.0.tar.gz", hash = "sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb", size = 19943, upload-time = "2025-12-21T09:29:43.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/c4/813bb09f0985cb21e959f21f2464169eca882656849adf727ac7bb7e1767/jaraco_functools-4.4.0-py3-none-any.whl", hash = "sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176", size = 10481, upload-time = "2025-12-21T09:29:42.27Z" }, +] + [[package]] name = "jedi" version = "0.19.2" @@ -630,6 +697,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" }, ] +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -664,6 +740,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + [[package]] name = "lxml" version = "5.4.0" @@ -752,6 +845,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "more-itertools" +version = "11.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/f7/139d22fef48ac78127d18e01d80cf1be40236ae489769d17f35c3d425293/more_itertools-11.0.2.tar.gz", hash = "sha256:392a9e1e362cbc106a2457d37cabf9b36e5e12efd4ebff1654630e76597df804", size = 144659, upload-time = "2026-04-09T15:01:33.297Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/98/6af411189d9413534c3eb691182bff1f5c6d44ed2f93f2edfe52a1bbceb8/more_itertools-11.0.2-py3-none-any.whl", hash = "sha256:6e35b35f818b01f691643c6c611bc0902f2e92b46c18fffa77ae1e7c46e912e4", size = 71939, upload-time = "2026-04-09T15:01:32.21Z" }, +] + [[package]] name = "numpy" version = "2.4.4" @@ -781,6 +883,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/6a/7e345032cc60501721ef94e0e30b60f6b0bd601f9174ebd36389a2b86d40/numpy-2.4.4-cp313-cp313t-win_arm64.whl", hash = "sha256:0dfd3f9d3adbe2920b68b5cd3d51444e13a10792ec7154cd0a2f6e74d4ab3233", size = 10292002, upload-time = "2026-03-29T13:20:25.909Z" }, ] +[[package]] +name = "open-mastr" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "keyring" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "psycopg2-binary" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "sqlalchemy" }, + { name = "tqdm" }, + { name = "xmltodict" }, + { name = "zeep" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/95/572681f6146523e5bb300bd843b290aeb537a5887eaa887a40adddd69be9/open_mastr-0.17.0.tar.gz", hash = "sha256:c9e5b6ff3a9cfdb5c5d0f0859b0985a6a5a65c5445a481428dffb5239e590d5d", size = 862367, upload-time = "2026-04-02T15:23:15.516Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/84/c978bd632ee36ee12a6477de395cd6dc68f98e66cfd5fab8a8c82e936d9d/open_mastr-0.17.0-py3-none-any.whl", hash = "sha256:f2388a74226e662c5ff7b6443fe9e64fe1f0cc27888b10d71028a67b9544df86", size = 154035, upload-time = "2026-04-02T15:23:13.441Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -856,6 +979,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/f0/c81e05b613866b76d2d1066490adf1a3dbc4ee9d9c839961c3fc8a6997af/pip-26.0.1-py3-none-any.whl", hash = "sha256:bdb1b08f4274833d62c1aa29e20907365a2ceb950410df15fc9521bad440122b", size = 1787723, upload-time = "2026-02-05T02:20:16.416Z" }, ] +[[package]] +name = "platformdirs" +version = "4.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/4a/0883b8e3802965322523f0b200ecf33d31f10991d0401162f4b23c698b42/platformdirs-4.9.6.tar.gz", hash = "sha256:3bfa75b0ad0db84096ae777218481852c0ebc6c727b3168c1b9e0118e458cf0a", size = 29400, upload-time = "2026-04-09T00:04:10.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/a6/a0a304dc33b49145b21f4808d763822111e67d1c3a32b524a1baf947b6e1/platformdirs-4.9.6-py3-none-any.whl", hash = "sha256:e61adb1d5e5cb3441b4b7710bea7e4c12250ca49439228cc1021c00dcfac0917", size = 21348, upload-time = "2026-04-09T00:04:09.463Z" }, +] + [[package]] name = "plumbum" version = "1.10.0" @@ -907,6 +1039,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/bf/2086963c69bdac3d7cff1cc7ff79b8ce5ea0bec6797a017e1be338a46248/protobuf-6.33.5-py3-none-any.whl", hash = "sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02", size = 170687, upload-time = "2026-01-29T21:51:32.557Z" }, ] +[[package]] +name = "psycopg2-binary" +version = "2.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c", size = 379620, upload-time = "2025-10-10T11:14:48.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c", size = 3756572, upload-time = "2025-10-10T11:12:32.873Z" }, + { url = "https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee", size = 3864529, upload-time = "2025-10-10T11:12:36.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0", size = 4411242, upload-time = "2025-10-10T11:12:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/10/04/6ca7477e6160ae258dc96f67c371157776564679aefd247b66f4661501a2/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766", size = 4468258, upload-time = "2025-10-10T11:12:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7e/6a1a38f86412df101435809f225d57c1a021307dd0689f7a5e7fe83588b1/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3", size = 4166295, upload-time = "2025-10-10T11:12:52.525Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/c07374c501b45f3579a9eb761cbf2604ddef3d96ad48679112c2c5aa9c25/psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f", size = 3983133, upload-time = "2025-10-30T02:55:24.329Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/993b7104cb8345ad7d4516538ccf8f0d0ac640b1ebd8c754a7b024e76878/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4", size = 3652383, upload-time = "2025-10-10T11:12:56.387Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/eaeb6029362fd8d454a27374d84c6866c82c33bfc24587b4face5a8e43ef/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c", size = 3298168, upload-time = "2025-10-10T11:13:00.403Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/50c3facc66bded9ada5cbc0de867499a703dc6bca6be03070b4e3b65da6c/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60", size = 3044712, upload-time = "2025-10-30T02:55:27.975Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8e/b7de019a1f562f72ada81081a12823d3c1590bedc48d7d2559410a2763fe/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1", size = 3347549, upload-time = "2025-10-10T11:13:03.971Z" }, + { url = "https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa", size = 2714215, upload-time = "2025-10-10T11:13:07.14Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -1062,6 +1213,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1147,6 +1307,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "requests-file" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/f8/5dc70102e4d337063452c82e1f0d95e39abfe67aa222ed8a5ddeb9df8de8/requests_file-3.0.1.tar.gz", hash = "sha256:f14243d7796c588f3521bd423c5dea2ee4cc730e54a3cac9574d78aca1272576", size = 6967, upload-time = "2025-10-20T18:56:42.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d5/de8f089119205a09da657ed4784c584ede8381a0ce6821212a6d4ca47054/requests_file-3.0.1-py2.py3-none-any.whl", hash = "sha256:d0f5eb94353986d998f80ac63c7f146a307728be051d4d1cd390dbdb59c10fa2", size = 4514, upload-time = "2025-10-20T18:56:41.184Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + [[package]] name = "rich" version = "14.3.3" @@ -1209,6 +1393,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" }, ] +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + [[package]] name = "sentry-sdk" version = "2.58.0" @@ -1267,6 +1464,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, ] +[[package]] +name = "sqlalchemy" +version = "2.0.49" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, + { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, + { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, + { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, + { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, + { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, +] + [[package]] name = "stack-data" version = "0.6.3" @@ -1290,6 +1513,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, ] +[[package]] +name = "tqdm" +version = "4.67.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, +] + [[package]] name = "traitlets" version = "5.14.3" @@ -1372,6 +1607,7 @@ dependencies = [ { name = "google-cloud-bigquery" }, { name = "lxml" }, { name = "numpy" }, + { name = "open-mastr" }, { name = "pandas" }, { name = "pydantic" }, { name = "python-slugify" }, @@ -1406,6 +1642,7 @@ requires-dist = [ { name = "google-cloud-bigquery", specifier = "~=3.40" }, { name = "lxml", specifier = "~=5.3" }, { name = "numpy", specifier = "~=2.4" }, + { name = "open-mastr", specifier = "~=0.17" }, { name = "pandas", specifier = "~=2.3" }, { name = "pydantic", specifier = "~=2.9.2" }, { name = "python-slugify", specifier = "~=8.0" }, @@ -1427,6 +1664,34 @@ dev = [ { name = "ruff", specifier = "~=0.15.6" }, ] +[[package]] +name = "xmltodict" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, +] + +[[package]] +name = "zeep" +version = "4.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "isodate" }, + { name = "lxml" }, + { name = "platformdirs" }, + { name = "pytz" }, + { name = "requests" }, + { name = "requests-file" }, + { name = "requests-toolbelt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/06/4f1d3ff61e930163565fb73616c6251e412a4d2fc7ed18214e1c2107258d/zeep-4.3.2.tar.gz", hash = "sha256:1a23a667ce9d73a0dbfdf15745bfa2b7ab0b6402135c0cd5067574838398e0e6", size = 166687, upload-time = "2025-09-15T10:26:03.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/78/f43f3feb70d67cbe260ec5b682ecc3c1850c8f437f1df707495126e51817/zeep-4.3.2-py3-none-any.whl", hash = "sha256:ed08c3179709172bfaaa9b76a6a545f8a57043ec6218e64e9deb81ff1e0ff79b", size = 101853, upload-time = "2025-09-15T10:26:02.12Z" }, +] + [[package]] name = "zipp" version = "3.23.0"