resolve conflicts

Author: TushigBili

.github/CONTRIBUTING.md

@@ -1,23 +1,37 @@
-# Contributing to Ithaca Transit Backend
+# Contributing to Score Backend
+
 👍🎉 First off, congrats on getting put on this pod 😂🎉👍
 
 The following is a set of guidelines for contributing to our backend. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
 # Versioning
+
 This is _extremely_ important as handling our backend can get quite messy.
 
 # Making PRs
+
 We want our PRs to be concise but informative. Some pointers as per our PR template:
+
 ### Title
+
 Summarize what changes you've made in one sentence. For example: "Exclude staff from the check for follows". For stacked PRs, please indicate clearly in the title where in the stack you are. For example: "[Eatery Refactor][4/5] Converted all files to MVP model."
+
 ### Overview
+
 Summarize generally what the purpose of this PR is.
+
 ### Changes
+
 Include details of what your changes actually are and how it is intended to work.
+
 ### Test Coverage
+
 Describe how you tested this feature: manual testing and/or unit testing. Please include repro steps and/or how to turn the feature on if applicable. In the context of this repo, add a plan for how you intend to test on [integration](https://github.com/cuappdev/integration), with your newly created issue linked.
+
 ### Next Steps
+
 If this applies, describe how you plan on addressing future PRs if this is a part of a multi-PR change.
+
 ### Related PRs or Issues
-If this applies, list related PRs against other branches or repositories.
 
+If this applies, list related PRs against other branches or repositories.

.github/workflows/deploy-dev.yml

@@ -6,39 +6,49 @@ on:
   workflow_dispatch:
 
 jobs:
-  path-context:
+  build:
     runs-on: ubuntu-latest
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Get SHA
         id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Docker Build & Push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: cornellappdev/score-dev:${{ steps.vars.outputs.sha_short }}
+          cache-from: type=registry,ref=cornellappdev/score-dev:cache
+          cache-to: type=registry,ref=cornellappdev/score-dev:cache,mode=max
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Remote SSH and Deploy
-        uses: appleboy/ssh-action@master
+        uses: appleboy/ssh-action@v1.0.3
         env:
-          IMAGE_TAG: ${{ steps.vars.outputs.sha_short }}
+          IMAGE_TAG: ${{ needs.build.outputs.sha_short }}
         with:
           host: ${{ secrets.DEV_SERVER_HOST }}
           username: ${{ secrets.SERVER_USERNAME }}
           key: ${{ secrets.DEV_SERVER_KEY }}
+          envs: IMAGE_TAG
           script: |
-            export IMAGE_TAG=${{ steps.vars.outputs.sha_short }}
-            docker stack rm thestack
-            sleep 20s
             docker stack deploy -c docker-compose.yml thestack
-            yes | docker system prune -a
+            docker service update --image cornellappdev/score-dev:${IMAGE_TAG} thestack_app
+            docker system prune -af

.github/workflows/deploy-prod.yml

@@ -6,39 +6,49 @@ on:
   workflow_dispatch:
 
 jobs:
-  path-context:
+  build:
     runs-on: ubuntu-latest
+    outputs:
+      sha_short: ${{ steps.vars.outputs.sha_short }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Get SHA
         id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short release)"
+        run: echo "sha_short=$(git rev-parse --short release)" >> $GITHUB_OUTPUT
       - name: Docker Build & Push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           push: true
           tags: cornellappdev/score:${{ steps.vars.outputs.sha_short }}
+          cache-from: type=registry,ref=cornellappdev/score:cache
+          cache-to: type=registry,ref=cornellappdev/score:cache,mode=max
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Remote SSH and Deploy
-        uses: appleboy/ssh-action@master
+        uses: appleboy/ssh-action@v1.0.3
         env:
-          IMAGE_TAG: ${{ steps.vars.outputs.sha_short }}
+          IMAGE_TAG: ${{ needs.build.outputs.sha_short }}
         with:
           host: ${{ secrets.PROD_SERVER_HOST }}
           username: ${{ secrets.SERVER_USERNAME }}
           key: ${{ secrets.PROD_SERVER_KEY }}
+          envs: IMAGE_TAG
           script: |
-            export IMAGE_TAG=${{ steps.vars.outputs.sha_short }}
-            docker stack rm thestack
-            sleep 20s
             docker stack deploy -c docker-compose.yml thestack
-            docker system prune -a
+            docker service update --image cornellappdev/score:${IMAGE_TAG} thestack_app
+            docker system prune -af

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.9
+FROM python:3.9-slim
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 COPY . .
 RUN pip3 install --upgrade pip
 RUN pip install -r requirements.txt
-CMD python app.py
+CMD gunicorn app:app -b 0.0.0.0:8000 --workers 1 --timeout 60 --max-requests 1000 --max-requests-jitter 200

app.py

@@ -1,22 +1,70 @@
 import logging
 import argparse
-from flask import Flask
+from flask import Flask, request, g
+import time
 from flask_graphql import GraphQLView
-from flask_apscheduler import APScheduler
 from graphene import Schema
 from src.schema import Query, Mutation
 from src.scrapers.games_scraper import fetch_game_schedule
 from src.scrapers.youtube_stats import fetch_videos
 from src.scrapers.daily_sun_scrape import fetch_news
 from src.services.article_service import ArticleService
 from src.utils.team_loader import TeamLoader
+import signal
+import sys
+from dotenv import load_dotenv
+
+load_dotenv()
 
 app = Flask(__name__)
 
-# Set up the scheduler
-scheduler = APScheduler()
-scheduler.init_app(app)
-scheduler.start()
+
+@app.before_request
+def start_timer():
+    g.start = time.time()
+
+    if request.path == "/graphql" and request.method == "POST":
+        try:
+            # Try to extract the GraphQL query name for better logging
+            query_data = request.get_json()
+            if query_data and "query" in query_data:
+                g.query = query_data["query"].split("{", 1)[0].strip()
+                logging.info(
+                    f"[{time.strftime('%H:%M:%S')}] --> GraphQL {g.query} started"
+                )
+        except:
+            pass
+
+    logging.info(
+        f"[{time.strftime('%H:%M:%S')}] --> {request.method} {request.path} started"
+    )
+
+
+@app.after_request
+def log_response_time(response):
+    if hasattr(g, "start"):
+        duration = time.time() - g.start
+
+        if duration > 5.0:  # Flag slow requests
+            if hasattr(g, "query"):
+                logging.warning(
+                    f"[{time.strftime('%H:%M:%S')}] <-- SLOW GraphQL {g.query} ({duration:.2f}s)"
+                )
+            else:
+                logging.warning(
+                    f"[{time.strftime('%H:%M:%S')}] <-- SLOW {request.method} {request.path} ({duration:.2f}s)"
+                )
+        else:
+            if hasattr(g, "query"):
+                logging.info(
+                    f"[{time.strftime('%H:%M:%S')}] <-- GraphQL {g.query} finished in {duration:.2f}s"
+                )
+            else:
+                logging.info(
+                    f"[{time.strftime('%H:%M:%S')}] <-- {request.method} {request.path} finished in {duration:.2f}s"
+                )
+    return response
+
 
 # Configure logging
 logging.basicConfig(
@@ -27,13 +75,16 @@
 
 schema = Schema(query=Query, mutation=Mutation)
 
+
 def create_context():
-    return {
-        "team_loader": TeamLoader()
-    }
+    return {"team_loader": TeamLoader()}
+
 
 app.add_url_rule(
-    "/graphql", view_func=GraphQLView.as_view("graphql", schema=schema, graphiql=True, get_context=create_context)
+    "/graphql",
+    view_func=GraphQLView.as_view(
+        "graphql", schema=schema, graphiql=True, get_context=create_context
+    ),
 )
 
 # Setup command line arguments
@@ -52,22 +103,35 @@ def parse_args():
     return parser.parse_args()
 
 args = parse_args()
+
+def signal_handler(sig, frame):
+    sys.exit(0)
+
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+# Only run scraping tasks if not disabled
 if not args.no_scrape:
-    @scheduler.task("interval", id="scrape_schedules", seconds=3600)
+    from flask_apscheduler import APScheduler
+    scheduler = APScheduler()
+    scheduler.init_app(app)
+    scheduler.start()
 
+    @scheduler.task("interval", id="scrape_schedules", seconds=43200) # 12 hours
     def scrape_schedules():
         logging.info("Scraping game schedules...")
         fetch_game_schedule()
 
-    @scheduler.task("interval", id="scrape_schedules", seconds=43200) # 12 hours
+    @scheduler.task("interval", id="scrape_videos", seconds=43200) # 12 hours
     def scrape_videos():
         logging.info("Scraping YouTube videos...")
         fetch_videos()
 
     scrape_schedules()
     scrape_videos()
 
-if not args.no_daily_sun:
+if not args.no_daily_sun and not args.no_scrape:
     @scheduler.task("interval", id="scrape_daily_sun", seconds=3600)
     def scrape_daily_sun():
         logging.info("Getting Daily Sun Sports News...")
@@ -83,4 +147,4 @@ def cleanse_daily_sun_db():
 
 
 if __name__ == "__main__":
-    app.run(debug=True, host="0.0.0.0", port=8000)
+    app.run(debug=True, host="0.0.0.0", port=8000)

docker-compose.yml

@@ -8,3 +8,10 @@ services:
       - "8000:8000"
     volumes:
       - ./ca-certificate.crt:/etc/ssl/ca-certificate.crt:ro # Mount MongoDB cert inside the container, ro for read only
+
+  scraper:
+    image: cornellappdev/score-dev:${IMAGE_TAG}
+    env_file: .env
+    command: python scraper.py
+    volumes:
+      - ./ca-certificate.crt:/etc/ssl/ca-certificate.crt:ro

requirements.txt

@@ -6,4 +6,6 @@ beautifulsoup4
 requests
 pillow
 Flask-APScheduler
-python-dotenv
+python-dotenv
+pytz
+gunicorn

scraper.py

@@ -0,0 +1,54 @@
+import logging
+import time
+import signal
+import sys
+from apscheduler.schedulers.background import BackgroundScheduler
+from src.scrapers.games_scraper import fetch_game_schedule
+from src.scrapers.youtube_stats import fetch_videos
+
+logging.basicConfig(
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+scheduler = BackgroundScheduler(daemon=True)
+
+
+def scrape_schedules():
+    start_time = time.time()
+    logging.info("Starting scraping games")
+    fetch_game_schedule()
+    elapsed_time = time.time() - start_time
+    logging.info(f"Completed scraping games in {elapsed_time:.2f} seconds")
+
+
+def scrape_videos():
+    start_time = time.time()
+    logging.info("Scraping YouTube videos")
+    fetch_videos()
+    elapsed_time = time.time() - start_time
+    logging.info(f"Completed scraping videos in {elapsed_time:.2f} seconds")
+
+
+def signal_handler(sig, frame):
+    logging.info("Shutting down scheduler...")
+    scheduler.shutdown(wait=True)
+    sys.exit(0)
+
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+if __name__ == "__main__":
+    scheduler.add_job(
+        scrape_schedules, "interval", seconds=60 * 60, id="scrape_schedules"
+    )
+    scheduler.add_job(
+        scrape_videos, "interval", seconds=60 * 60 * 24, id="scrape_videos"
+    )
+    scheduler.start()
+    scrape_schedules()
+    scrape_videos()
+
+    signal.pause()