refactor: add endpoints

- `app.py`: add all and quotes endpoints
- `crawler.py`: move to `bin`, skip db creation and only sanitize data
- `json_to_sqlite.py`: convert `db.json` to `db.sqlite` for faster access
- `db.json`: remove trailing comma in author name (randomized order in the process)
- `docker-compose.yml`: mount working directory
- `Dockerfile`: add timezone, reorder layerf for better caching, add env vars, split entrypoint and command

Author: pythoninthegrass

Dockerfile

@@ -2,12 +2,35 @@
 
 FROM python:3.11-slim-buster
 
+ENV TZ=America/Chicago
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
 WORKDIR /app
 
-COPY . .
+COPY requirements.txt .
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONFAULTHANDLER 1
+ENV PYTHONUNBUFFERED 1
 
-RUN pip install --no-cache-dir -r requirements.txt
+ENV PIP_DISABLE_PIP_VERSION_CHECK=on
+ENV PIP_DEFAULT_TIMEOUT=100
+
+RUN python -m pip install --no-cache-dir -r requirements.txt
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+ENV WEB_CONCURRENCY=2
+
+COPY . .
 
 EXPOSE 8000
 
-CMD ["gunicorn", "-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "app:app"]
+ENTRYPOINT [ "gunicorn" ]
+CMD ["-k", "uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:8000", "app:app", "--reload"]

app.py

@@ -1,19 +1,46 @@
 #!/usr/bin/env python
 
-from fastapi import FastAPI
-from fastapi import Request
-from fastapi.responses import HTMLResponse
+import sqlite3
+from fastapi import FastAPI, Query,Request
+from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
+from typing import List, Optional
 
 app = FastAPI()
 templates = Jinja2Templates(directory="templates")
 
 
-@app.get('/hello')
-def hello():
-    return 'Hello, World!'
-
-
 @app.get('/')
 def home(request: Request):
     return templates.TemplateResponse("index.html", {"request": request})
+
+
+# TODO: fix optional name parameter
+@app.get('/hello/{name}')
+def hello(name: Optional[str] = None) -> JSONResponse:
+    if name:
+        message = f"Hello, {name}!"
+    else:
+        message = f"Hello, World!"
+    return JSONResponse(content={"hello": message})
+
+
+# TODO: improve sql performance
+@app.get('/all')
+def get_quotes(request: Request) -> JSONResponse:
+    db = sqlite3.connect("db.sqlite")
+    c = db.cursor()
+    c.execute("SELECT * FROM quotes")
+    quotes = c.fetchall()
+    db.close()
+    return JSONResponse(content={"quotes": quotes})
+
+
+@app.get('/quotes/{limit}')
+def get_quotes_by_limit(request: Request, limit: int) -> JSONResponse:
+    db = sqlite3.connect("db.sqlite")
+    c = db.cursor()
+    c.execute(f"SELECT * FROM quotes ORDER BY RANDOM() LIMIT {limit}")
+    quotes = c.fetchall()
+    db.close()
+    return JSONResponse(content={"quotes": quotes})

crawler.py bin/crawler.pycrawler.py renamed to bin/crawler.py

@@ -8,15 +8,14 @@
 from decouple import config
 from tinydb import TinyDB, Query
 from urllib.parse import urljoin
-from tinydb import TinyDB, Query
 
-#  env vars
+# env vars
 name = config("NAME", default="goodreads")
 url = config("URL", default="https://www.goodreads.com/quotes")
 ttl = config("TTL", default=300)
 
 # database
-db = TinyDB("db.json")
+db = TinyDB("../db.json")
 
 # cache the requests to sqlite, expire after n time
 if not ttl:
@@ -79,6 +78,7 @@ def strip_characters(data):
                 item[key] = value.strip()
             elif key == "author":
                 value = value.strip()
+                value = value.rstrip(",")
                 item[key] = value
 
     return data
@@ -99,10 +99,51 @@ def sanitize_data(data):
     return valid_data
 
 
+def check_db():
+    """Check db.json for data."""
+
+    Quote = Query()
+    quotes = db.count(Quote.author.exists())
+
+    if quotes:
+        return True
+    else:
+        return False
+
+
+def update_db():
+    """Update existing db.json with normalized data."""
+
+    # get authors
+    authors_set = set(entry['author'] for entry in db.all())
+    quotes_set = set(entry['quote'] for entry in db.all())
+
+    # normalize authors
+    for author in authors_set:
+        normalized_author = strip_characters([{"author": author}])[0]["author"]
+        normalized_author_list = sanitize_data([{"author": normalized_author}])
+        if normalized_author_list:
+            normalized_author = normalized_author_list[0]["author"]
+            db.update({"author": normalized_author}, Author.author == author)
+
+    # normalize quotes
+    for quote in quotes_set:
+        normalized_quote = strip_characters([{"quote": quote}])[0]["quote"]
+        normalized_quote_list = sanitize_data([{"quote": normalized_quote}])
+        if normalized_quote_list:
+            normalized_quote = normalized_quote_list[0]["quote"]
+            db.update({"quote": normalized_quote}, Quote.quote == quote)
+
+
 async def main():
     # ! drop the table (qa)
     # db.drop_tables()
 
+    # check if there is data in the db
+    if check_db():
+        update_db()
+        return
+
     # initialize cursor with the starting URL
     cursor = url
 

bin/json_to_sqlite.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+import json
+import sqlite3
+from pathlib import Path
+
+raw_db = Path("../db.json")
+db = Path("../db.sqlite")
+
+# ! remove the db if it exists (qa)
+if db.exists():
+    db.unlink()
+
+if not db.exists():
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    c.execute(
+        "CREATE TABLE quotes (id INTEGER PRIMARY KEY, quote TEXT, author TEXT)"
+    )
+    conn.commit()
+    conn.close()
+
+with open(raw_db) as f:
+    data = json.load(f)
+
+quotes = set()
+
+quotes = [(key, value['quote'],
+           value['author']) for key, value in data['_default'].items()
+]
+
+conn = sqlite3.connect(db)
+c = conn.cursor()
+c.executemany("INSERT INTO quotes VALUES (?, ?, ?)", quotes)
+conn.commit()
+conn.close()