refresh-docs.yml

name: Refresh docs

# Weekly re-scrape of the MaStR registry, re-render of the marimo notebooks,
# regeneration of all sample SVGs + the wind animation GIF, then commit the
# updated artifacts back to main so GitHub Pages picks them up.

on:
  schedule:
    # Sundays at 03:17 UTC. Picked off-the-hour to dodge cron stampedes.
    - cron: "17 3 * * 0"
  workflow_dispatch:
    inputs:
      pages_to_scrape_pv:
        description: "Number of 25k-row pages of top-sorted PV plants to fetch"
        default: "8"
      pages_to_scrape_wind:
        description: "Number of 25k-row pages of non-PV plants to fetch"
        default: "7"

permissions:
  contents: write

concurrency:
  group: refresh-docs
  cancel-in-progress: false

jobs:
  refresh:
    runs-on: ubuntu-latest
    timeout-minutes: 60

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Install pixi
        uses: prefix-dev/setup-pixi@v0.8.10
        with:
          pixi-version: latest
          cache: true

      - name: Resolve env
        run: pixi install

      - name: Refresh open-mastr SQLite snapshot (wind, solar, storage, storage_units, market)
        # ~6 GB DB + 2 GB ZIP fit comfortably on the runner; drop the ZIP after
        # parse to keep disk pressure off the subsequent render steps.
        # SQLITE_DATABASE_PATH (read by open-mastr/helpers.py:50) directs the
        # write to the repo path so mastr_db.DB_PATH resolves to _REPO_DB.
        env:
          SQLITE_DATABASE_PATH: ${{ github.workspace }}/data/mastr/open-mastr.db
        run: |
          mkdir -p data/mastr
          pixi run db-mastr-core
          rm -f ~/.open-MaStR/data/xml_download/*.zip || true

      - name: Fetch Kreis polygons
        run: pixi run fetch-kreise

      - name: Scrape MaStR (non-PV — wind, hydro, biomass…)
        env:
          PAGES_NON_PV: ${{ github.event.inputs.pages_to_scrape_wind || '7' }}
        run: |
          mkdir -p data
          for p in $(seq 1 "${PAGES_NON_PV}"); do
            curl --get \
              'https://www.marktstammdatenregister.de/MaStR/Einheit/EinheitJson/GetErweiterteOeffentlicheEinheitStromerzeugung' \
              --data-urlencode 'sort=' \
              --data-urlencode "page=${p}" \
              --data-urlencode 'pageSize=25000' \
              --data-urlencode 'group=' \
              --data-urlencode "filter=Energieträger~neq~'2495'~and~Energieträger~neq~'2496'" \
              --data-urlencode 'forExport=true' \
              -o "data/data-${p}.json" &
          done
          wait

      - name: Scrape MaStR (BESS top-200k)
        run: |
          mkdir -p data-bess
          for p in $(seq 1 8); do
            curl --get \
              'https://www.marktstammdatenregister.de/MaStR/Einheit/EinheitJson/GetErweiterteOeffentlicheEinheitStromerzeugung' \
              --data-urlencode 'sort=Bruttoleistung-desc' \
              --data-urlencode "page=${p}" \
              --data-urlencode 'pageSize=25000' \
              --data-urlencode 'group=' \
              --data-urlencode "filter=Energieträger~eq~'2496'" \
              --data-urlencode 'forExport=true' \
              -o "data-bess/data-${p}.json" &
          done
          wait

      - name: Scrape MaStR (PV top-200k)
        env:
          PAGES_PV: ${{ github.event.inputs.pages_to_scrape_pv || '8' }}
        run: |
          mkdir -p data-pv
          for p in $(seq 1 "${PAGES_PV}"); do
            curl --get \
              'https://www.marktstammdatenregister.de/MaStR/Einheit/EinheitJson/GetErweiterteOeffentlicheEinheitStromerzeugung' \
              --data-urlencode 'sort=Bruttoleistung-desc' \
              --data-urlencode "page=${p}" \
              --data-urlencode 'pageSize=25000' \
              --data-urlencode 'group=' \
              --data-urlencode "filter=Energieträger~eq~'2495'" \
              --data-urlencode 'forExport=true' \
              -o "data-pv/data-${p}.json" &
          done
          wait

      - name: Re-render sample SVGs + Bundesland chart
        run: pixi run python scripts/render_samples.py

      - name: Re-render wind + PV + BESS animation GIFs + MP4s
        run: pixi run python scripts/render_wind_gif.py all

      - name: Rebuild per-Kreis JSON
        run: pixi run python scripts/build_kreise_json.py

      - name: Build bulk-download artefacts
        run: pixi run python scripts/build_downloads.py

      - name: Run unit tests
        run: pixi run test

      - name: Re-export marimo HTML
        run: pixi run docs-build

      - name: Commit refreshed docs
        run: |
          git config user.name  "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add fig/sample-*.svg \
                  fig/wind-*.gif fig/pv-*.gif fig/bess-*.gif \
                  fig/wind-*.mp4 fig/pv-*.mp4 fig/bess-*.mp4 \
                  docs/assets/sample-*.svg \
                  docs/assets/wind-*.gif docs/assets/pv-*.gif docs/assets/bess-*.gif \
                  docs/assets/wind-*.mp4 docs/assets/pv-*.mp4 docs/assets/bess-*.mp4 \
                  docs/assets/kreise.json \
                  docs/data/*.parquet docs/data/*.csv.gz \
                  docs/pv.html docs/wind.html || true
          if git diff --cached --quiet; then
            echo "No changes to commit."
          else
            git commit -m "Weekly refresh: re-scrape MaStR + re-render docs ($(date -u +%Y-%m-%d))"
            git push
          fi