feat: presigned-S3 download path for package blobs (#57)

API Gateway HTTP API + apig-wsgi base64-rounds binary responses and
doesn't stream them, which throttles downloads of the encrypted SQLite
to ~17KB/s (a 100MB package would take 100min). Bypass the bottleneck
by handing the client a presigned S3 URL.

Additive only — nothing existing is changed:

- New S3 bucket dumpus-prod-package-data (private, encrypted, 90d
  lifecycle, CORS for browser fetches). Worker uploads each finished
  encrypted blob via a VPC gateway endpoint so the bytes don't transit
  fck-nat.
- Worker dual-writes: still inserts the legacy DB row, ALSO uploads to
  S3. S3 failure is logged-and-ignored so the legacy /data path keeps
  working.
- New endpoint GET /process/<id>/blob returns
  {url, iv, ttl} — same auth contract as /data, returns 409 if the
  blob isn't in S3 yet (caller falls back to /data).
- /data is untouched. Frontend can migrate to /blob at its own pace,
  decrypt locally with the UPN, and we deprecate /data later.

Author: Androz2091

infra/terraform/blob_storage.tf

@@ -0,0 +1,81 @@
+# Stores encrypted package blobs out-of-band from Postgres so the API can
+# return short presigned URLs instead of streaming hundreds of MB through
+# Lambda + API Gateway (which is slow and capped at 6MB).
+#
+# The blob itself is encrypted client-key-derived AES-CBC by the worker, so
+# there's no privacy concern about the bucket holding it — only the holder
+# of the UPN can decrypt it.
+
+resource "aws_s3_bucket" "package_data" {
+  bucket = "${local.name}-package-data"
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "package_data" {
+  bucket = aws_s3_bucket.package_data.id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_public_access_block" "package_data" {
+  bucket = aws_s3_bucket.package_data.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+resource "aws_s3_bucket_versioning" "package_data" {
+  bucket = aws_s3_bucket.package_data.id
+  versioning_configuration {
+    status = "Disabled"
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "package_data" {
+  bucket = aws_s3_bucket.package_data.id
+
+  rule {
+    id     = "expire-blobs"
+    status = "Enabled"
+
+    filter {}
+
+    expiration {
+      days = var.package_data_retention_days
+    }
+
+    abort_incomplete_multipart_upload {
+      days_after_initiation = 1
+    }
+  }
+}
+
+# CORS so the frontend (web.dumpus.app and dev) can fetch the presigned URL
+# from the browser. Only GET is allowed.
+resource "aws_s3_bucket_cors_configuration" "package_data" {
+  bucket = aws_s3_bucket.package_data.id
+
+  cors_rule {
+    allowed_methods = ["GET"]
+    allowed_origins = ["*"]
+    allowed_headers = ["*"]
+    expose_headers  = ["ETag", "Content-Length"]
+    max_age_seconds = 300
+  }
+}
+
+# VPC gateway endpoint for S3 so worker uploads (potentially hundreds of MB)
+# don't go through fck-nat. Gateway endpoints are free, route table-based.
+resource "aws_vpc_endpoint" "s3" {
+  vpc_id            = aws_vpc.main.id
+  service_name      = "com.amazonaws.${var.region}.s3"
+  vpc_endpoint_type = "Gateway"
+  route_table_ids   = [aws_route_table.private.id]
+
+  tags = { Name = "${local.name}-s3-endpoint" }
+}

infra/terraform/iam.tf

@@ -41,6 +41,15 @@ data "aws_iam_policy_document" "api" {
       aws_secretsmanager_secret.wh_url.arn,
     ]
   }
+
+  # boto3.generate_presigned_url signs with this role's identity, so the
+  # role itself must be allowed to GetObject — the URL just borrows that
+  # permission for ~5 minutes.
+  statement {
+    sid       = "ReadPackageBlobs"
+    actions   = ["s3:GetObject"]
+    resources = ["${aws_s3_bucket.package_data.arn}/*"]
+  }
 }
 
 resource "aws_iam_role_policy" "api" {
@@ -87,6 +96,13 @@ data "aws_iam_policy_document" "worker" {
       aws_secretsmanager_secret.wh_url.arn,
     ]
   }
+
+  # Worker uploads each finished encrypted blob.
+  statement {
+    sid       = "WritePackageBlobs"
+    actions   = ["s3:PutObject"]
+    resources = ["${aws_s3_bucket.package_data.arn}/*"]
+  }
 }
 
 resource "aws_iam_role_policy" "worker" {

infra/terraform/lambda.tf

@@ -15,6 +15,10 @@ locals {
       DISWHO_JWT_SECRET = aws_secretsmanager_secret.diswho_jwt_secret.arn
       WH_URL            = aws_secretsmanager_secret.wh_url.arn
     })
+
+    # Encrypted package blobs live here; API generates presigned URLs.
+    PACKAGE_DATA_BUCKET                    = aws_s3_bucket.package_data.id
+    PACKAGE_DATA_PRESIGNED_URL_TTL_SECONDS = tostring(var.package_data_presigned_url_ttl_seconds)
   }
 }
 

infra/terraform/outputs.tf

@@ -37,3 +37,7 @@ output "fck_nat_eip" {
   description = "Public IP that all outbound Lambda traffic comes from"
   value       = aws_eip.fck_nat.public_ip
 }
+
+output "package_data_bucket" {
+  value = aws_s3_bucket.package_data.id
+}

infra/terraform/variables.tf

@@ -139,6 +139,18 @@ variable "worker_reserved_concurrency" {
 
 # ---- App secrets / config ----
 
+variable "package_data_retention_days" {
+  description = "How long to keep encrypted package blobs in S3 before lifecycle deletion. Acts as a passive GDPR-friendly cleanup."
+  type        = number
+  default     = 90
+}
+
+variable "package_data_presigned_url_ttl_seconds" {
+  description = "TTL on the presigned download URL the API hands the client. Short = safer if the URL leaks."
+  type        = number
+  default     = 300
+}
+
 variable "wh_url" {
   description = "Optional Discord webhook URL the app pings for internal notifications (new package, errored package, etc.). Leave blank to disable."
   type        = string

src/app.py

@@ -232,6 +232,56 @@ def get_package_data(package_id):
 
     return Response(data, mimetype='application/octet-stream')
 
+
+@app.route('/process/<package_id>/blob', methods=['GET'])
+def get_package_blob(package_id):
+    """Return a presigned S3 URL the client can fetch the encrypted blob from.
+
+    Pairs with /data: same auth, same lookup, but instead of streaming the
+    decrypted SQLite through API Gateway (slow + 6MB cap), the client
+    downloads encrypted bytes directly from S3 and decrypts them locally
+    using its UPN.
+
+    Response: {"url": <presigned GET>, "iv": <hex>, "ttl": <seconds>}
+    Errors: 401 (bad bearer), 404 (no row), 409 (row exists but no S3 blob).
+    """
+    import os
+    import blob_storage
+    from db import SavedPackageData
+
+    (is_auth, _) = check_authorization_bearer(request, package_id)
+    if not is_auth:
+        return make_response('', 401)
+
+    if not blob_storage.is_enabled():
+        # /blob is meaningless without S3 — fall through so callers can
+        # detect and switch back to /data.
+        return jsonify({'errorMessageCode': 'S3_NOT_CONFIGURED'}), 501
+
+    session = Session()
+    row = session.query(SavedPackageData).filter_by(package_id=package_id).order_by(
+        SavedPackageData.created_at.desc()
+    ).first()
+    session.close()
+
+    if not row:
+        return make_response('', 404)
+
+    if not blob_storage.exists(package_id):
+        # Row exists in DB but worker hadn't migrated yet (or upload failed).
+        # Caller can fall back to /data.
+        return jsonify({'errorMessageCode': 'BLOB_NOT_IN_S3'}), 409
+
+    ttl = int(os.getenv('PACKAGE_DATA_PRESIGNED_URL_TTL_SECONDS', '300'))
+    url = blob_storage.presigned_url(package_id, ttl_seconds=ttl)
+
+    iv_value = row.iv
+    if isinstance(iv_value, (bytes, bytearray)):
+        iv_value = iv_value.hex()
+
+    return jsonify({'url': url, 'iv': iv_value, 'ttl': ttl}), 200
+
+
 @app.route('/process/<package_id>', methods=['DELETE'])
 def cancel_package(package_id):
 

src/blob_storage.py

@@ -0,0 +1,61 @@
+"""S3-backed storage for encrypted package blobs.
+
+The worker uploads the encrypted SQLite directly to S3 (one object per
+package). The API generates short presigned GET URLs so clients can
+download directly from S3 — bypassing API Gateway, which is slow and
+6MB-capped for binary responses.
+
+No-op when PACKAGE_DATA_BUCKET is unset (local dev): callers fall back
+to the legacy DB-backed path.
+"""
+import os
+
+
+def _bucket():
+    return os.getenv("PACKAGE_DATA_BUCKET")
+
+
+def is_enabled() -> bool:
+    return bool(_bucket())
+
+
+def _key(package_id: str) -> str:
+    # Flat layout. Package IDs are MD5 hex digests, so no path-traversal risk.
+    return f"packages/{package_id}.bin"
+
+
+def upload_encrypted(package_id: str, encrypted_bytes: bytes) -> None:
+    """Push the encrypted blob to S3 under a deterministic key."""
+    import boto3
+
+    boto3.client("s3").put_object(
+        Bucket=_bucket(),
+        Key=_key(package_id),
+        Body=encrypted_bytes,
+        ContentType="application/octet-stream",
+    )
+
+
+def presigned_url(package_id: str, ttl_seconds: int = 300) -> str:
+    """Return a short-lived URL the client can GET directly from S3."""
+    import boto3
+
+    return boto3.client("s3").generate_presigned_url(
+        "get_object",
+        Params={"Bucket": _bucket(), "Key": _key(package_id)},
+        ExpiresIn=ttl_seconds,
+    )
+
+
+def exists(package_id: str) -> bool:
+    """True if a blob has been uploaded for this package."""
+    import boto3
+    from botocore.exceptions import ClientError
+
+    try:
+        boto3.client("s3").head_object(Bucket=_bucket(), Key=_key(package_id))
+        return True
+    except ClientError as e:
+        if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey", "NotFound"):
+            return False
+        raise

src/tasks.py

@@ -875,6 +875,17 @@ def read_analytics_file(package_status_id, package_id, link, session):
     key = extract_key_from_discord_link(link)
     (data, iv) = encrypt_sqlite_data(zipped_buffer, key)
 
+    # When PACKAGE_DATA_BUCKET is configured, also push the encrypted blob to
+    # S3 so /blob can serve a presigned URL. Failure here is non-fatal — the
+    # legacy DB blob still gets written below and the slow /data path keeps
+    # working.
+    import blob_storage
+    if blob_storage.is_enabled():
+        try:
+            blob_storage.upload_encrypted(package_id, data)
+        except Exception as e:
+            print(f'WARN: S3 blob upload failed for {package_id}: {e}')
+
     session.add(SavedPackageData(package_id=package_id, encrypted_data=data, iv=iv))
     session.commit()