Merge pull request #53 from cuappdev/master

Prod merge Jan 30

Author: JoshD94

.env_template

@@ -2,3 +2,4 @@ YOUTUBE_API_KEY=
 MONGO_URI=
 MONGO_DB=
 STAGE=
+DAILY_SUN_URL=

README.md

@@ -22,4 +22,6 @@ To start the project, run the following command in the terminal
 
 ## Setting up the database
 
-Add /graphql to the url to access the interactive GraphQL platform
+Create a Mongo database named `score_db` and another named `daily_sun_db`. A partnership with the Daily Sun has given us access to their articles which we copy and paginate the results for frontend.
+
+Add /graphql to the url to access the interactive GraphQL platform

app.py

@@ -5,6 +5,10 @@
 from flask_graphql import GraphQLView
 from graphene import Schema
 from src.schema import Query, Mutation
+from src.scrapers.games_scraper import fetch_game_schedule
+from src.scrapers.youtube_stats import fetch_videos
+from src.scrapers.daily_sun_scrape import fetch_news
+from src.services.article_service import ArticleService
 from src.utils.team_loader import TeamLoader
 import signal
 import sys
@@ -83,6 +87,30 @@ def create_context():
     ),
 )
 
+# Setup command line arguments
+def parse_args():
+    parser = argparse.ArgumentParser(description="Skip scraping tasks, for dev purposes.")
+    parser.add_argument(
+        "--no-scrape",
+        action="store_true",
+        help="Skips scraping tasks if set, useful for frontend development.",
+    )
+    parser.add_argument(
+        "--no-daily-sun",
+        action="store_true",
+        help="Skips using the Daily Sun page for alerts",
+    )
+    return parser.parse_args()
+
+# Only parse arguments when running directly (not when imported by gunicorn)
+if __name__ == "__main__":
+    args = parse_args()
+else:
+    # Default args when imported by gunicorn
+    class DefaultArgs:
+        no_scrape = False
+        no_daily_sun = False
+    args = DefaultArgs()
 
 def signal_handler(sig, frame):
     sys.exit(0)
@@ -91,5 +119,50 @@ def signal_handler(sig, frame):
 signal.signal(signal.SIGINT, signal_handler)
 signal.signal(signal.SIGTERM, signal_handler)
 
+# Only parse arguments when running directly (not when imported by gunicorn)
+if __name__ == "__main__":
+    args = parse_args()
+else:
+    # Default args when imported by gunicorn
+    class DefaultArgs:
+        no_scrape = False
+        no_daily_sun = False
+    args = DefaultArgs()
+
+# Only run scraping tasks if not disabled
+if not args.no_scrape:
+    from flask_apscheduler import APScheduler
+    scheduler = APScheduler()
+    scheduler.init_app(app)
+    scheduler.start()
+
+    @scheduler.task("interval", id="scrape_schedules", seconds=43200) # 12 hours
+    def scrape_schedules():
+        logging.info("Scraping game schedules...")
+        fetch_game_schedule()
+
+    @scheduler.task("interval", id="scrape_videos", seconds=43200) # 12 hours
+    def scrape_videos():
+        logging.info("Scraping YouTube videos...")
+        fetch_videos()
+
+    scrape_schedules()
+    scrape_videos()
+
+if not args.no_daily_sun and not args.no_scrape:
+    @scheduler.task("interval", id="scrape_daily_sun", seconds=3600)
+    def scrape_daily_sun():
+        logging.info("Getting Daily Sun Sports News...")
+        fetch_news()
+
+    @scheduler.task("interval", id="cleanse_daily_sun_db", seconds=604800) # 1 week
+    def cleanse_daily_sun_db():
+        logging.info("Cleaning the Daily Sun database from old articles...")
+        ArticleService.cleanse_old_articles()
+
+    scrape_daily_sun()
+    cleanse_daily_sun_db()
+
+
 if __name__ == "__main__":
     app.run(debug=True, host="0.0.0.0", port=8000)

src/database.py

@@ -48,6 +48,7 @@ def keep_connection_alive():
 
 # Access the database
 db = client[os.getenv("MONGO_DB", "score_db")]
+daily_sun_db = client[os.getenv("DAILY_SUN_DB", "daily_sun_db")]
 
 
 def setup_database_indexes():
@@ -65,6 +66,31 @@ def setup_database_indexes():
 
         # Index for sorting operations
         game_collection.create_index([("date", -1)], background=True)
+        
+        # Index to have unique games so we won't add duplicates
+        game_collection.create_index(
+            [
+                ("sport", 1),
+                ("gender", 1),
+                ("date", 1),
+                ("opponent_id", 1),
+                ("state", 1),
+            ],
+            unique=True,
+            background=True
+        )
+        
+        # Additional index for tournament games (without opponent_id)
+        game_collection.create_index(
+            [
+                ("sport", 1),
+                ("gender", 1),
+                ("date", 1),
+                ("city", 1),
+                ("state", 1),
+            ],
+            background=True
+        )
 
         print("✅ MongoDB indexes created successfully")
     except Exception as e:

src/models/__init__.py

@@ -1,3 +1,4 @@
 from .game import Game
 from .team import Team
-from .youtube_video import YoutubeVideo
+from .youtube_video import YoutubeVideo
+from .article import Article

src/models/article.py

@@ -0,0 +1,56 @@
+from bson.objectid import ObjectId
+from datetime import datetime
+
+class Article:
+    """
+    A model representing a news article.
+
+    Attributes:
+        - title: The title of the article
+        - image: The filename of the article's main image
+        - sports_type: The specific sport category
+        - published_at: The publication date
+        - url: The URL to the full article
+        - slug: Unique identifier from the source
+        - created_at: When the article was added to our DB
+    """
+    def __init__(self, title, sports_type, published_at, url, slug, image=None, id=None, created_at=None):
+        self.id = id if id else str(ObjectId())
+        self.title = title
+        self.image = image
+        self.sports_type = sports_type
+        self.published_at = published_at
+        self.url = url
+        self.slug = slug
+        self.created_at = created_at if created_at else datetime.now()
+
+    def to_dict(self):
+        """
+        Converts the Article object to a dictionary format for MongoDB storage.
+        """
+        return {
+            "_id": self.id,
+            "title": self.title,
+            "image": self.image,
+            "sports_type": self.sports_type,
+            "published_at": self.published_at,
+            "url": self.url,
+            "slug": self.slug,
+            "created_at": self.created_at
+        }
+
+    @staticmethod
+    def from_dict(data):
+        """
+        Converts a MongoDB document to an Article object.
+        """
+        return Article(
+            id=data.get("_id"),
+            title=data.get("title"),
+            image=data.get("image"),
+            sports_type=data.get("sports_type"),
+            published_at=data.get("published_at"),
+            url=data.get("url"),
+            slug=data.get("slug"),
+            created_at=data.get("created_at")
+        )

src/models/game.py

@@ -17,6 +17,7 @@ class Game:
         - `time`            The time of the game. (optional)
         - `box_score`       The scoring summary of the game (optional)
         - `score_breakdown` The scoring breakdown of the game (optional)
+        - 'ticket_link'    The ticket link for the game (optional)
     """
 
     def __init__(
@@ -35,6 +36,7 @@ def __init__(
         score_breakdown=None,
         team=None,
         utc_date=None,
+        ticket_link=None,
     ):
         self.id = id if id else str(ObjectId())
         self.city = city
@@ -50,6 +52,7 @@ def __init__(
         self.score_breakdown = score_breakdown
         self.team = team
         self.utc_date = utc_date
+        self.ticket_link = ticket_link
 
     def to_dict(self):
         """
@@ -70,6 +73,7 @@ def to_dict(self):
             "score_breakdown": self.score_breakdown,
             "team": self.team,
             "utc_date": self.utc_date,
+            "ticket_link": self.ticket_link,
         }
 
     @staticmethod
@@ -92,4 +96,5 @@ def from_dict(data) -> None:
             score_breakdown=data.get("score_breakdown"),
             team=data.get("team"),
             utc_date=data.get("utc_date"),
+            ticket_link=data.get("ticket_link"),
         )

src/models/youtube_video.py

@@ -12,10 +12,11 @@ class YoutubeVideo:
         - `thumbnail`       The thumbnail of the video, as a URL string pointing to a `.jpg` file.
         - `url`             The URL of the video.
         - `published_at`    The date and time the video was published.
+        - `duration`        The duration of the video.
     """
 
     def __init__(
-        self, title, description, thumbnail, b64_thumbnail, url, published_at, id=None
+        self, title, description, thumbnail, b64_thumbnail, url, published_at, duration=None, id=None
     ):
         self.id = id if id else str(ObjectId())
         self.title = title
@@ -24,6 +25,7 @@ def __init__(
         self.b64_thumbnail = b64_thumbnail
         self.url = url
         self.published_at = published_at
+        self.duration = duration
 
     def to_dict(self):
         """
@@ -37,6 +39,7 @@ def to_dict(self):
             "b64_thumbnail": self.b64_thumbnail,
             "url": self.url,
             "published_at": self.published_at,
+            "duration": self.duration,
         }
 
     @staticmethod
@@ -52,4 +55,5 @@ def from_dict(data):
             b64_thumbnail=data.get("b64_thumbnail"),
             url=data.get("url"),
             published_at=data.get("published_at"),
+            duration=data.get("duration"),
         )

src/mutations/__init__.py

@@ -1,3 +1,4 @@
 from .create_game import CreateGame
 from .create_team import CreateTeam
-from .create_youtube_video import CreateYoutubeVideo
+from .create_youtube_video import CreateYoutubeVideo
+from .create_article import CreateArticle

src/mutations/create_article.py

@@ -0,0 +1,26 @@
+from graphene import Mutation, String, Field
+from src.types import ArticleType
+from src.services.article_service import ArticleService
+
+class CreateArticle(Mutation):
+    class Arguments:
+        title = String(required=True)
+        sports_type = String(required=True)
+        published_at = String(required=True)
+        url = String(required=True)
+        slug = String(required=True)
+        image = String(required=False)
+
+    article = Field(lambda: ArticleType)
+
+    def mutate(self, info, title, sports_type, published_at, url, slug, image=None):
+        article_data = {
+            "title": title,
+            "sports_type": sports_type,
+            "published_at": published_at,  # Already in ISO 8601 format
+            "url": url,
+            "slug": slug,
+            "image": image
+        }
+        new_article = ArticleService.create_article(article_data)
+        return CreateArticle(article=new_article)