Refactored models, added Google Sheets scraping (#119)

* Model Changes

* Add gym and FC hours

* Implemented Amenities

* Add capacities scraper

* Add Pool Hours

* Add Court Hours

* Fixed timezone issues

* Added scheduling

* Update README.md

* Update README.md

* Update README.md

* Compacted regular hours

* Add bowling hours

* Added scraper for special hours

* Update sp_hours_scraper.py

* Address PR comments

Author: vinnie4k

.github/workflows/test.yml

@@ -26,4 +26,4 @@ jobs:
         pip install --force-reinstall setuptools==44.0.0
         pip freeze
         pip install -r requirements.txt
-        python tests.py
+        python -m unittest src.tests.test_scraper

.gitignore

@@ -16,4 +16,5 @@ migrations
 .vscode
 Archive
 scripts
-*.sqlite3
+*.sqlite3
+service-account-key.json

README.md

@@ -2,19 +2,28 @@
 
 Technologies involved include:
 1. Flask
-2. GraphQL
+2. GraphQL (Graphene)
 
-## Virtualenv
+## Setup Instructions
 
-Virtualenv setup!
+### Database
+1. Install PostgreSQL on your local computer.
+2. Create a database and user:
+```
+createdb uplift
+createuser -P -s -e local
+```
+- This creates a database called `uplift` and a user called `local`. You will then use these when defining your environment variables.
+
+### Virtualenv
 
 ```bash
 virtualenv venv
 source venv/bin/activate
 pip install -r requirements.txt
 ```
 
-## Environment Variables
+### Environment Variables
 It's recommended to use [`autoenv`](https://github.com/kennethreitz/autoenv).
 The required environment variables for this API are the following:
 
@@ -34,15 +43,19 @@ To use `autoenv` with this repository, run the following and set the variables a
 cp env.template .env
 ````
 
-## Setting Up the Database with Data
+### Service Account Key
+Check the `#uplift-backend` channel for the `service-account-key.json` which should be placed in the root directory.
+
+## Running the App and Test Suite
+
+To run the app:
 ````bash
-python setup_db.py
+python app.py
 ````
 
-## Running the App
-
+To run the test suite:
 ````bash
-flask run
+python -m unittest src.tests.test_scraper
 ````
 
 ## Setting up linter

app.py

@@ -1,13 +1,16 @@
+import logging, schedule, time
 from flask import Flask, render_template
 from flask_graphql import GraphQLView
 from graphene import Schema
 from graphql.utils import schema_printer
 from src.database import db_session, init_db
+from src.models.capacity import Capacity
+from src.models.openhours import OpenHours
 from src.schema import Query
-from src.constants import create_gym_table
-from src.scrapers.scraper import scrape_classes
-from src.scrapers.gym_scraper import scrape_times
-from src.scrapers.scraper import scrape_classes, scrape_pool_hours
+from src.scrapers.capacities_scraper import fetch_capacities
+from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility
+from src.scrapers.sp_hours_scraper import fetch_sp_facility
+from src.utils.utils import create_gym_table
 
 
 app = Flask(__name__)
@@ -29,12 +32,22 @@ def shutdown_session(exception=None):
     db_session.remove()
 
 
-# Create database and fill it with constants
+def scrape_sheets():
+    logging.info("Scraping from sheets...")
+
+    # Fetch Hours
+    fetch_reg_facility()
+    fetch_reg_building()
+    fetch_sp_facility()
+
+    # Fetch Capacities
+    fetch_capacities()
+
+
+# Create database and fill it with data
 init_db()
 create_gym_table()
-scrape_times()
-scrape_classes(3)
-scrape_pool_hours()
+scrape_sheets()
 
 # Create schema.graphql
 with open("schema.graphql", "w+") as schema_file:
@@ -44,3 +57,9 @@ def shutdown_session(exception=None):
 # Should only be used for dev
 if __name__ == "__main__":
     app.run(host="127.0.0.1", port=5000)  # For Dev Purposes only (use start_server.sh for release)
+
+# Schedule the scraping to run every 10 minutes
+schedule.every(10).minutes.do(scrape_sheets)
+while True:
+    schedule.run_pending()
+    time.sleep(60)

requirements.txt

@@ -21,13 +21,14 @@ Flask-GraphQL==2.0.0
 Flask-Migrate==2.4.0
 Flask-Script==2.0.5
 Flask-SQLAlchemy==2.3.1
-google-auth==1.6.3
+google-auth==1.12.0
 graphene==2.1.3
 graphene-sqlalchemy==2.3.0
 graphql-core==2.1
 graphql-relay==0.4.5
 graphql-server-core==1.1.1
 greenlet==2.0.2
+gspread==5.12.3
 gunicorn==19.9.0
 identify==2.5.24
 idna==2.6
@@ -41,6 +42,7 @@ MarkupSafe==1.1.1
 marshmallow==3.0.0rc4
 marshmallow-sqlalchemy==0.16.2
 nodeenv==1.8.0
+pandas==2.1.4
 parso==0.8.3
 platformdirs==3.10.0
 pre-commit==1.18.3
@@ -53,10 +55,12 @@ pyasn1==0.5.0
 pyasn1-modules==0.3.0
 python-dateutil==2.8.0
 python-editor==1.0.4
+pytz==2023.3
 PyYAML==6.0
 requests==2.28.2
 rsa==4.0
 Rx==1.6.1
+schedule==1.1.0
 singledispatch==3.7.0
 six==1.11.0
 snapshottest==0.6.0

schema.graphql

@@ -2,70 +2,71 @@ schema {
   query: Query
 }
 
-type Capacity {
+type Amenity {
   id: ID!
-  facilityId: Int!
-  count: Int!
-  percent: Float!
-  updated: DateTime!
+  gymId: Int!
+  type: AmenityType!
 }
 
-type Class {
-  id: ID!
-  name: String!
-  description: String!
-  gyms: [ClassInstance]
+enum AmenityType {
+  SHOWERS
+  LOCKERS
+  PARKING
+  ELEVATORS
 }
 
-type ClassInstance {
+type Capacity {
   id: ID!
-  gymId: Int
-  classId: Int!
-  location: String!
-  instructor: String!
-  isCanceled: Boolean!
-  isVirtual: Boolean!
-  startTime: DateTime
-  endTime: DateTime
-  class_: Class
-  gym: Gym
+  count: Int!
+  facilityId: Int!
+  percent: Float!
+  updated: Int!
 }
 
-scalar DateTime
+enum CourtType {
+  BASKETBALL
+  VOLLEYBALL
+  BADMINTON
+}
 
 type Facility {
   id: ID!
+  facilityType: FacilityType!
   gymId: Int!
   name: String!
-  facilityType: FacilityType!
-  openHours(name: String): [OpenHours]
-  capacities: [Capacity]
   capacity: Capacity
+  hours: [OpenHours]
 }
 
 enum FacilityType {
   FITNESS
   POOL
+  BOWLING
+  COURT
 }
 
 type Gym {
   id: ID!
-  name: String!
-  description: String!
-  location: String!
+  address: String!
+  imageUrl: String
   latitude: Float!
   longitude: Float!
-  imageUrl: String
+  name: String!
+  amenities: [Amenity]
   facilities: [Facility]
-  classes: [ClassInstance]
+  hours: [OpenHours]
 }
 
 type OpenHours {
   id: ID!
-  facilityId: Int!
-  day: Int!
-  startTime: String
-  endTime: String
+  courtType: CourtType
+  endTime: Int!
+  facilityId: Int
+  gymId: Int
+  isShallow: Boolean
+  isSpecial: Boolean!
+  isWomen: Boolean
+  startTime: Int!
 }
 
 type Query {