crawl_proxy_blocks.yml

name: Crawl Wikipedia Proxy Blocks

on:
  repository_dispatch:
  workflow_dispatch:
  schedule:
    # Run daily at 00:22 UTC
    - cron: '22 0 * * *'

jobs:
  crawl:
    runs-on: ubuntu-latest
    permissions:
      contents: write
      actions: write
    timeout-minutes: 120
    # Only run on the original repository, not forks
    if: github.repository == 'networkcats/OpenProxyDB'

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.14'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt

      - name: Fetch data files from data branch
        run: |
          git fetch origin data:data 2>/dev/null || true
          if git show data:last_crawl_time.txt >/dev/null 2>&1; then
            git show data:last_crawl_time.txt > last_crawl_time.txt
            echo "Loaded last_crawl_time.txt from data branch"
          else
            echo "No last_crawl_time.txt found in data branch"
          fi
          if git show data:block_metadata.csv >/dev/null 2>&1; then
            git show data:block_metadata.csv > block_metadata.csv
            echo "Loaded block_metadata.csv from data branch"
          else
            echo "No block_metadata.csv found in data branch"
          fi

      - name: Run crawler
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python -m src.crawler

      - name: Set tag name
        run: |
          echo "TAG_NAME=$(date +"%Y.%m.%d")" >> $GITHUB_ENV

      - name: Push data files to data branch
        run: |
          git config user.name "${{ github.actor }}"
          git config user.email "${{ github.actor }}@users.noreply.github.com"
          
          # Save the file contents
          CRAWL_TIME=$(cat last_crawl_time.txt)
          METADATA_CSV=$(cat block_metadata.csv)
          
          # Remove the files to avoid checkout conflict
          rm -f last_crawl_time.txt block_metadata.csv
          
          # Check if data branch exists remotely
          if git ls-remote --exit-code --heads origin data >/dev/null 2>&1; then
            # Branch exists, fetch and checkout
            git fetch origin data
            git checkout -B data origin/data
          else
            # Create orphan branch
            git checkout --orphan data
            git rm -rf . >/dev/null 2>&1 || true
          fi
          
          # Write the files
          echo "$CRAWL_TIME" > last_crawl_time.txt
          echo "$METADATA_CSV" > block_metadata.csv
          
          # Commit with amend to keep single commit
          git add last_crawl_time.txt block_metadata.csv
          if git rev-parse HEAD >/dev/null 2>&1; then
            git commit --amend -m "Update data files"
          else
            git commit -m "Update data files"
          fi
          
          # Force push
          git push --force origin data

      - name: Upload to Releases
        uses: softprops/action-gh-release@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ env.TAG_NAME }}
          name: Proxy Blocks ${{ env.TAG_NAME }}
          body: |
            Proxy block list updated on ${{ env.TAG_NAME }}
            
            - New IPs added: ${{ env.IPS_ADDED }}
            - IPs removed: ${{ env.IPS_REMOVED }}
            - Total IPs: ${{ env.TOTAL_IPS }}
            
            This release contains IP addresses blocked by Wikipedia for proxy-related reasons.
          files: proxy_blocks.csv
            
      - name: Remove old Releases
        uses: dev-drprasad/delete-older-releases@master
        with:
          keep_latest: 2
          delete_tags: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Delete old workflow runs
        uses: Mattraks/delete-workflow-runs@main
        with:
          retain_days: 0
          keep_minimum_runs: 2