big_things.yml

name: Multi-Job Workflow

on:
  # schedule:
   # - cron: '0 12,16,21,4 * * *'  # 12 PM, 4 PM, 9 PM, 4 AM UTC
  workflow_dispatch:  # Allows manual triggering of the workflow

permissions:
  contents: write
  issues: write

jobs:
  setup:
    name: Setup Environment
    runs-on: ubuntu-latest
    steps:
    - name: Checkout repository
      uses: actions/checkout@v3
      with:
        clean: true  # Ensure a clean slate
        fetch-depth: 0
        fetch-tags: true

    - name: Configure Git
      run: |
        git config --global user.email "your-email@example.com"
        git config --global user.name "Your Name"

    - name: Install Python and Node.js
      run: |
        python -m pip install --upgrade pip
        pip install requests beautifulsoup4 ckanapi markdownify
        npm install puppeteer node-fetch@2

    - name: Install Rename Utility
      run: sudo apt-get update && sudo apt-get install -y rename

    - name: Sanitize File Paths
      run: |
        find . -name '*[":,<>|*?]*' -exec rename 's/[^a-zA-Z0-9_.-]/_/g' {} +

    - name: Upload Repository as Artifact
      uses: actions/upload-artifact@v3
      with:
        name: repository
        path: ${{ github.workspace }}
        exclude: |
          node_modules/**  # Exclude node_modules
          **/node_modules/**  # Exclude nested node_modules directories
          .git/**  # Optionally exclude .git directory
          **/*.log  # Optionally exclude log files
        if-no-files-found: warn


  scrape:
    name: Scrape Data
    needs: setup
    runs-on: ubuntu-latest
    steps:
    - name: Download Repository Artifact
      uses: actions/download-artifact@v3
      with:
        name: repository

    - name: Run scraper
      run: |
        python scraper.py
        python REF_STDs/scraper.py

    - name: Commit and Push Scraped Data
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        git add docs/scraped_table_en.csv docs/scraped_table_fr.csv docs/scraped_content_eng.html docs/scraped_content_fra.html *.atom REF_STDs/
        git commit -m "Update scraped data and content" || echo "No changes to commit"
        git push

    - name: Upload Scrape Results as Artifact
      uses: actions/upload-artifact@v3
      with:
        name: scrape-results
        path: docs/

  detect_changes:
    name: Detect Changes
    needs: scrape
    runs-on: ubuntu-latest
    outputs:
      changed: ${{ steps.check-commit-message.outputs.changed }}
    steps:
    - name: Download Repository Artifact
      uses: actions/download-artifact@v3
      with:
        name: repository

    - name: Check Commit Message
      id: check-commit-message
      run: |
        COMMIT_MESSAGE=$(git log -1 --pretty=%B)
        echo "Last commit message: $COMMIT_MESSAGE"
        if [[ "$COMMIT_MESSAGE" == *"Update scraped data and content"* ]]; then
          echo "::set-output name=changed::true"
        else
          echo "::set-output name=changed::false"
        fi

    - name: Take Screenshots if Changes Detected
      if: ${{ steps.check-commit-message.outputs.changed == 'true' }}
      run: |
        node -e "
        const puppeteer = require('puppeteer');
        (async () => {
          const browser = await puppeteer.launch();
          const page = await browser.newPage();
          await page.goto('https://www.canada.ca/en/government/system/digital-government/digital-government-innovations/enabling-interoperability/gc-enterprise-data-reference-standards.html');
          await page.screenshot({ path: 'docs/screenshot_en.png', fullPage: true });
          await page.goto('https://www.canada.ca/fr/gouvernement/systeme/gouvernement-numerique/innovations-gouvernementales-numeriques/permettre-interoperabilite/normes-referentielles-pangouvernementales-relatives-donnees-gc.html');
          await page.screenshot({ path: 'docs/screenshot_fr.png', fullPage: true });
          await browser.close();
        })();
        "

    - name: Commit and Push Screenshots
      if: ${{ steps.check-commit-message.outputs.changed == 'true' }}
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        git add docs/screenshot_en.png docs/screenshot_fr.png
        git commit -m "Add updated screenshots for data change detection" || echo "No changes to commit"
        git push

  create_issue:
    name: Create GitHub Issue
    needs: detect_changes
    runs-on: ubuntu-latest
    if: ${{ needs.detect_changes.outputs.changed == 'true' }}
    steps:
    - name: Create Issue Body
      run: |
        echo "# Data Change Detected" > issue_body.md
        echo "The data on the monitored website has changed." >> issue_body.md
        echo "![Screenshot EN](https://github.com/PatLittle/GC-Ref-Data-Tracker/blob/main/docs/screenshot_en.png?raw=true)" >> issue_body.md
        echo "![Screenshot FR](https://github.com/PatLittle/GC-Ref-Data-Tracker/blob/main/docs/screenshot_fr.png?raw=true)" >> issue_body.md

    - name: Create Issue
      uses: peter-evans/create-issue-from-file@v4
      with:
        title: "Data Change Detected on Website"
        content-filepath: issue_body.md
        labels: update, screenshot
        assignees: your-github-username

  fetch_feeds:
    name: Fetch Feeds
    needs: scrape
    runs-on: ubuntu-latest
    steps:
    - name: Download Repository Artifact
      uses: actions/download-artifact@v3
      with:
        name: repository

    - name: Run Feed Fetcher
      run: python atom_feed_fetcher.py

    - name: Commit and Push Feed Results
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        git add -A
        git commit --allow-empty -m "Update fetched feeds" || echo "No changes to commit"
        git push

  fetch_usage:
    name: Fetch Usage
    needs: scrape
    runs-on: ubuntu-latest
    steps:
    - name: Download Repository Artifact
      uses: actions/download-artifact@v3
      with:
        name: repository

    - name: Run Usage Fetcher
      run: python portal_usage_fetcher.py

    - name: Commit and Push Usage Results
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        git add -A
        git commit --allow-empty -m "Update usage data" || echo "No changes to commit"
        git push

  fetch_commits:
    name: Fetch Commits
    needs: fetch_feeds
    runs-on: ubuntu-latest
    steps:
    - name: Download Repository Artifact
      uses: actions/download-artifact@v3
      with:
        name: repository

    - name: Fetch GitHub Commits
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        node fetch_commits.js
        git add -A
        git commit --allow-empty -m "Update commit data" || echo "No changes to commit"
        git push