scraper.yml

name: stock scraper
on:
  # schedule:
  #   - cron: "0,30 13-21 * * 1-5"
  workflow_dispatch:

jobs:
  scraping:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up python
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: pip install -r requirements.txt

      - name: Run scraper with retry
        run: |
          for i in {1..3}; do
            echo "Attempt${i}"
            python main.py && break || sleep 10
          done


      - name: Push csv to data branch
        run: |
          git config --global user.name "github-actions"
          git config --global user.email "github-actions@gmail.com"
          git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}
          git fetch origin data-store
          git checkout data-store
          git add *.csv
          git commit -m "Scraped at $(date -u)" || echo "No changes to commit"
          git push origin data-store