sites_data_pipeline.yml

name: Sites Data Pipeline (7-step)

# ── Trigger ───────────────────────────────────────────────────────────────────
on:
  workflow_dispatch:
    inputs:

      site_url:
        description: 'CKAN site URL to collect metadata for (e.g. https://data.gov.au)'
        required: true
        type: string

# ── Job ───────────────────────────────────────────────────────────────────────
jobs:
  pipeline:
    name: Run Sites Data Pipeline
    runs-on: ubuntu-latest
    timeout-minutes: 360
    permissions:
      contents: write

    defaults:
      run:
        working-directory: sites-data-fetch

    steps:

      # ── Setup ──────────────────────────────────────────────────────────────
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: pip
          cache-dependency-path: sites-data-fetch/requirements.txt

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      # ── Build seed CSV from the single URL input ───────────────────────────
      - name: Create seed 0.csv
        run: |
          echo "url" > 0.csv
          echo "${{ inputs.site_url }}" >> 0.csv
          echo "✅ Created 0.csv for: ${{ inputs.site_url }}"

      # ── Step 1: Name Processing ────────────────────────────────────────────
      - name: "Step 1 — Name Processing"
        run: |
          echo "▶  Step 1: Extracting instance name"
          python 1-nameProcess.py
          echo "✅ Step 1 complete"

      # ── Step 2: CKAN API Stats ─────────────────────────────────────────────
      - name: "Step 2 — CKAN API Stats"
        run: |
          echo "▶  Step 2: Fetching stats via CKAN Action API"
          python 2-CKANActionAPI.py
          echo "✅ Step 2 complete"

      # ── Step 3: Site Type Classification ──────────────────────────────────
      - name: "Step 3 — Site Type Classification"
        run: |
          echo "▶  Step 3: Classifying site type"
          python 3-siteType.py
          echo "✅ Step 3 complete"

      # ── Step 4: About Page Descriptions ───────────────────────────────────
      - name: "Step 4 — About Page Description"
        run: |
          echo "▶  Step 4: Scraping About page"
          python 4-description.py
          echo "✅ Step 4 complete"

      # ── Step 5: Location Analysis (LLM) ───────────────────────────────────
      - name: "Step 5 — Location Analysis (LLM)"
        env:
          OPEN_ROUTER_KEY: ${{ secrets.OPEN_ROUTER_KEY }}
        run: |
          if [ -z "$OPEN_ROUTER_KEY" ]; then
            echo "❌ OPEN_ROUTER_KEY secret is not set."
            exit 1
          fi
          echo "▶  Step 5: Inferring geographic location via LLM"
          python 5-locationAnalyser.py
          echo "✅ Step 5 complete"

      # ── Step 6: Geocoding ──────────────────────────────────────────────────
      - name: "Step 6 — Geocoding"
        run: |
          echo "▶  Step 6: Geocoding location to lat/lon"
          python 6-geocode.py
          echo "✅ Step 6 complete"

      # ── Step 7: Timestamp ──────────────────────────────────────────────────
      - name: "Step 7 — Add Timestamp"
        run: |
          echo "▶  Step 7: Stamping output with UTC date"
          python 7-tstamp.py 6.csv 7.csv
          echo "✅ Step 7 complete"

      # ── Upload final output as artifact ───────────────────────────────────
      - name: Upload final output (7.csv)
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: sites-metadata-${{ github.run_id }}
          path: sites-data-fetch/7.csv
          retention-days: 30

      # ── Summary ────────────────────────────────────────────────────────────
      - name: Workflow summary
        if: always()
        run: |
          echo "=== Pipeline Summary ==="
          echo "Status:    ${{ job.status }}"
          echo "Site URL:  ${{ inputs.site_url }}"
          echo "Timestamp: $(date -u)"
          echo ""
          echo "Output files:"
          for f in 1.csv 2.csv 3.csv 4.csv 5.csv 6.csv 7.csv; do
            [ -f "$f" ] && echo "  $f — $(wc -l < $f) lines" || echo "  $f — not produced"
          done