chore: drop /data endpoint and server-side decryption (#64)

Author: Androz2091

README.md

@@ -160,14 +160,33 @@ Available steps:
 
 ### Fetch a package data
 
-* `GET /process/<package_id>/data`
+* `GET /process/<package_id>/blob`
 
-Response: the Discord Data Package SQLite database (GZIP of the binary SQLite file), decrypted.
+Returns a short-lived presigned S3 URL the client downloads the encrypted
+SQLite from directly. Decryption happens client-side using the UPN as the
+key, so the encryption key never reaches the server.
+
+Response:
+```json
+{
+    "url": "https://<bucket>.s3.<region>.amazonaws.com/...",
+    "iv": "abc123...",
+    "ttl": 300
+}
+```
+
+`iv` is a hex string for real packages, `null` for the demo (the demo blob
+is unencrypted). Decrypt with `AES-CBC` using `SHA-256(UPN)` as the key
+and the returned IV; the plaintext is a gzipped SQLite file.
 
 Status codes:
-* `200`: the data is available and has been returned.
+* `200`: presigned URL returned.
 * `401`: the UPN KEY provided in the Authorization header is not valid.
-* `404`: unknown package ID.
+* `404`: no blob for this package id.
+* `501`: server doesn't have the S3 backend wired up.
+
+Demo (`/process/demo/blob`) is unauthenticated and lazy-seeds the blob on
+the first call after deploy.
 
 ### Fetch a package user
 

src/app.py

@@ -1,14 +1,14 @@
 import socket
 
-from flask import Flask, jsonify, request, Response, make_response
+from flask import Flask, jsonify, request, make_response
 from flask_cors import CORS
 
 from flask_limiter import Limiter
 
 # Import tasks before db so dotenv loads before SQLAlchemy reads POSTGRES_URL.
 import tasks  # noqa: F401
 from enqueue import enqueue_package
-from db import PackageProcessStatus, SavedPackageData, Session, fetch_package_status, fetch_package_data, fetch_package_rank
+from db import PackageProcessStatus, SavedPackageData, Session, fetch_package_status, fetch_package_rank
 
 from sqlite import generate_demo_database
 
@@ -195,44 +195,6 @@ def get_package_status(package_id):
     return jsonify(res), 200
 
 
-@app.route('/process/<package_id>/data', methods=['GET'])
-def get_package_data(package_id):
-
-    if package_id == 'demo':
-        session = Session()
-        data = fetch_package_data('demo', 'demo', session)
-        if not data:
-            binary_data = generate_demo_database()
-            session.add(SavedPackageData(package_id='demo', encrypted_data=binary_data, iv='demo'))
-            session.commit()
-            data = binary_data
-        session.close()
-        return Response(data, mimetype='application/octet-stream')
-
-    (is_auth, auth_upn) = check_authorization_bearer(request, package_id)
-    if not is_auth:
-        return make_response('', 401)
-
-    session = Session()
-    data = fetch_package_data(package_id, auth_upn, session)
-    session.close()
-
-    if not data:
-        return make_response('', 404)
-
-    send_internal_notification({
-        'embeds': [
-            {
-                'title': 'Fetching existing package data',
-                'description': f'Package ID: {package_id}',
-                'color': 0xaea4e0
-            }
-        ]
-    })
-    
-    return Response(data, mimetype='application/octet-stream')
-
-
 @app.route('/process/<package_id>/blob', methods=['GET'])
 def get_package_blob(package_id):
     """Return a presigned S3 URL the client can fetch the package blob from.
@@ -367,11 +329,3 @@ def page_not_found(e):
 @app.errorhandler(500)
 def internal_server_error(e):
     return jsonify({'error': 'Internal server error.'}), 500
-
-def rm_demo():
-    session = Session()
-    session.query(SavedPackageData).filter(SavedPackageData.package_id == 'demo').delete()
-    session.commit()
-    session.close()
-
-rm_demo()

src/crypto.py

@@ -27,22 +27,3 @@ def encrypt_sqlite_data(sqlite_buffer, auth_upn):
 
     return (encrypted_data, iv)
 
-def decrypt_sqlite_data(encrypted_data, iv, auth_upn):
-
-    fixed_size_upn = hashlib.sha256(auth_upn.encode()).digest()
-
-    if not isinstance(iv, bytes):
-        iv = bytes.fromhex(iv[2:])
-
-    # Prepare the decryptor
-    cipher = Cipher(algorithms.AES(fixed_size_upn), modes.CBC(iv), backend=default_backend())
-    decryptor = cipher.decryptor()
-
-    # Decrypt the data
-    decrypted_data = decryptor.update(encrypted_data) + decryptor.finalize()
-
-    # Remove the padding
-    unpadder = padding.PKCS7(128).unpadder()
-    unpadded_data = unpadder.update(decrypted_data) + unpadder.finalize()
-
-    return unpadded_data

src/db.py

@@ -1,8 +1,7 @@
-from crypto import decrypt_sqlite_data
 import os
 
 from datetime import datetime
-from sqlalchemy import create_engine, Column, Integer, String, DateTime, LargeBinary, Boolean
+from sqlalchemy import create_engine, Column, Integer, String, DateTime, Boolean
 from sqlalchemy.orm import declarative_base, sessionmaker
 from sqlalchemy.sql import func, text
 
@@ -15,7 +14,6 @@ class SavedPackageData(Base):
 
     id = Column(Integer, primary_key=True)
     package_id = Column(String(255), nullable=False)
-    encrypted_data = Column(LargeBinary(), nullable=False)
     iv = Column(String(255), nullable=False)
     created_at = Column(DateTime, nullable=False, default=func.now())
     updated_at = Column(DateTime, nullable=False, onupdate=func.now(), default=func.now())
@@ -41,6 +39,12 @@ class PackageProcessStatus(Base):
 
 Base.metadata.create_all(engine)
 
+# One-shot migration: drop the legacy encrypted_data blob column on the
+# saved_package_data table. Encrypted bytes live in S3 now; only iv stays
+# in the DB. IF EXISTS makes this idempotent across cold starts.
+with engine.begin() as conn:
+    conn.execute(text("ALTER TABLE saved_package_data DROP COLUMN IF EXISTS encrypted_data;"))
+
 Session = sessionmaker(bind=engine)
 
 def update_progress (package_status_id, package_id, current_progress, session):
@@ -89,16 +93,6 @@ def fetch_package_status(package_id, session):
     status = session.query(PackageProcessStatus).filter_by(package_id=package_id).order_by(PackageProcessStatus.created_at.desc()).first()
     return status
 
-def fetch_package_data(package_id, auth_upn, session):
-    result = session.query(SavedPackageData).filter_by(package_id=package_id).order_by(SavedPackageData.created_at.desc()).first()
-    if result:
-        if package_id == 'demo':
-            return result.encrypted_data
-        encrypted_data = result.encrypted_data
-        iv = result.iv
-        sqlite_buffer = decrypt_sqlite_data(encrypted_data, iv, auth_upn)
-        return sqlite_buffer
-
 def fetch_pending_packages():
     session = Session()
     # select * from package_process_status pps where step <> 'PROCESSED' and error_message_code is null

src/tasks.py

@@ -875,18 +875,11 @@ def read_analytics_file(package_status_id, package_id, link, session):
     key = extract_key_from_discord_link(link)
     (data, iv) = encrypt_sqlite_data(zipped_buffer, key)
 
-    # When PACKAGE_DATA_BUCKET is configured, also push the encrypted blob to
-    # S3 so /blob can serve a presigned URL. Failure here is non-fatal — the
-    # legacy DB blob still gets written below and the slow /data path keeps
-    # working.
+    # Encrypted blob → S3 (clients fetch via presigned URL); only the iv
+    # stays in Postgres so the API can hand it to the client for decryption.
     import blob_storage
-    if blob_storage.is_enabled():
-        try:
-            blob_storage.upload(package_id, data)
-        except Exception as e:
-            print(f'WARN: S3 blob upload failed for {package_id}: {e}')
-
-    session.add(SavedPackageData(package_id=package_id, encrypted_data=data, iv=iv))
+    blob_storage.upload(package_id, data)
+    session.add(SavedPackageData(package_id=package_id, iv=iv))
     session.commit()
 
     print(f'SQLite serialization: {time.time() - start}')