feat: move worker to Fargate, drop the 15-min Lambda cap (#74)

Author: Androz2091

.github/workflows/deploy.yml

@@ -19,9 +19,11 @@ permissions:
 
 env:
   AWS_REGION: eu-west-1
-  ECR_REPO: dumpus-prod-lambda
+  ECR_LAMBDA_REPO: dumpus-prod-lambda
+  ECR_WORKER_REPO: dumpus-prod-worker
   API_FUNCTION: dumpus-prod-api
-  WORKER_FUNCTION: dumpus-prod-worker
+  FORWARDER_FUNCTION: dumpus-prod-forwarder
+  WORKER_TASK_FAMILY: dumpus-prod-worker
 
 jobs:
   build-and-deploy:
@@ -53,29 +55,41 @@ jobs:
       - name: Set up Buildx
         uses: docker/setup-buildx-action@v4
 
-      - name: Build & push Lambda image
-        id: build
+      - name: Build & push Lambda image (api + forwarder)
         uses: docker/build-push-action@v7
         with:
           context: .
           file: Dockerfile.lambda
           push: true
           # Lambda only runs amd64 unless the function is configured for arm64.
-          # Keep amd64 unless we explicitly switch the function architecture.
           platforms: linux/amd64
           provenance: false
           tags: |
-            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }}
-            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:latest
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_LAMBDA_REPO }}:${{ steps.meta.outputs.image_tag }}
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_LAMBDA_REPO }}:latest
           cache-from: type=gha,scope=lambda
           cache-to: type=gha,scope=lambda,mode=max
 
-      - name: Update Lambda functions
+      - name: Build & push worker image (Fargate)
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile.worker
+          push: true
+          platforms: linux/amd64
+          provenance: false
+          tags: |
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_WORKER_REPO }}:${{ steps.meta.outputs.image_tag }}
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_WORKER_REPO }}:latest
+          cache-from: type=gha,scope=worker
+          cache-to: type=gha,scope=worker,mode=max
+
+      - name: Update Lambdas
         env:
-          IMAGE_URI: ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }}
+          IMAGE_URI: ${{ steps.ecr.outputs.registry }}/${{ env.ECR_LAMBDA_REPO }}:${{ steps.meta.outputs.image_tag }}
         run: |
           set -euo pipefail
-          for fn in "${API_FUNCTION}" "${WORKER_FUNCTION}"; do
+          for fn in "${API_FUNCTION}" "${FORWARDER_FUNCTION}"; do
             echo "::group::Updating $fn"
             aws lambda update-function-code \
               --function-name "$fn" \
@@ -86,6 +100,38 @@ jobs:
             echo "::endgroup::"
           done
 
+      - name: Register new worker task definition revision
+        env:
+          IMAGE_URI: ${{ steps.ecr.outputs.registry }}/${{ env.ECR_WORKER_REPO }}:${{ steps.meta.outputs.image_tag }}
+        run: |
+          set -euo pipefail
+          # Pull the active revision, swap in the new image, register a new
+          # revision. ECS resolves task family → latest revision on each
+          # runTask call, so no further wiring is needed.
+          aws ecs describe-task-definition \
+            --task-definition "$WORKER_TASK_FAMILY" \
+            --query 'taskDefinition' \
+            > /tmp/td.json
+          jq --arg img "$IMAGE_URI" '
+            .containerDefinitions[0].image = $img
+            | {
+                family,
+                taskRoleArn,
+                executionRoleArn,
+                networkMode,
+                containerDefinitions,
+                requiresCompatibilities,
+                cpu,
+                memory,
+                ephemeralStorage,
+                runtimePlatform
+              }
+            | with_entries(select(.value != null))
+          ' /tmp/td.json > /tmp/td-new.json
+          aws ecs register-task-definition \
+            --cli-input-json file:///tmp/td-new.json \
+            > /dev/null
+
       - name: Smoke test API
         run: |
           set -euo pipefail

Dockerfile.worker

@@ -0,0 +1,21 @@
+# Fargate task image. Plain Python — no Lambda runtime — because Fargate
+# runs ordinary container processes. Keeps the image small and the entrypoint
+# obvious.
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# System deps are minimal: pandas wheels are manylinux, orjson too. If a
+# future package adds something native, install it here.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Flat layout: matches Dockerfile.lambda so existing top-level imports
+# (`from tasks import ...`) keep working.
+COPY src/ /app/
+
+ENTRYPOINT ["python", "-u", "worker_main.py"]

README.md

@@ -47,17 +47,18 @@ By default, Dumpus API will only treat zip files sent from `https://discord.clic
 
 ## Deploy to AWS
 
-A Terraform stack under `infra/terraform/` provisions a serverless deployment of the API on AWS:
+A Terraform stack under `infra/terraform/` provisions a deployment of the API on AWS:
 
 | Component        | AWS service              |
 | ---------------- | ------------------------ |
 | API              | Lambda (container image) behind API Gateway HTTP API |
-| Worker           | Lambda triggered by SQS                              |
+| Forwarder        | Lambda triggered by SQS, fires one Fargate task per message |
+| Worker           | Fargate task (no time / memory caps, pay-per-run)    |
 | Database         | RDS Postgres in private subnets                      |
 | Outbound NAT     | fck-nat instance (NAT Gateway replacement)           |
-| Secrets          | Secrets Manager + Lambda env                         |
+| Secrets          | Secrets Manager + Lambda/task env                    |
 | TLS / DNS        | ACM cert + Route53 alias to API Gateway              |
-| CI               | GitHub OIDC role; build → ECR → `update-function-code` |
+| CI               | GitHub OIDC role; build → ECR → `update-function-code` + `register-task-definition` |
 
 ### Bootstrap
 
@@ -69,21 +70,23 @@ A Terraform stack under `infra/terraform/` provisions a serverless deployment of
 
 ### Day-to-day deploys
 
-Push to `main` → `.github/workflows/deploy.yml` builds the container, pushes to ECR tagged with the git SHA, and rolls both Lambdas via OIDC. No long-lived AWS keys in GitHub.
+Push to `main` → `.github/workflows/deploy.yml` builds both container images (Lambda for the API + forwarder, plain Python for the Fargate worker), pushes them to ECR tagged with the git SHA, rolls both Lambdas, and registers a new ECS task definition revision. The next runTask call picks up the new image. No long-lived AWS keys in GitHub.
 
 ### Operations
 
 ```bash
 aws logs tail /aws/lambda/<name-prefix>-<env>-api --follow
-aws logs tail /aws/lambda/<name-prefix>-<env>-worker --follow
+aws logs tail /aws/lambda/<name-prefix>-<env>-forwarder --follow
+aws logs tail /aws/ecs/<name-prefix>-<env>-worker --follow
 aws sqs receive-message --queue-url "$(terraform output -raw sqs_dlq_url)"
 ```
 
 Things to keep in mind:
 
 - **API cold start** is a few seconds while pandas imports. Invisible on the async submit/poll flow; use provisioned concurrency if a sync endpoint must be sub-second.
-- **Worker `/tmp` cap** defaults to 5GB (max 10GB). Bump `worker_ephemeral_storage_mb` if users upload very large Discord exports.
-- **Worker timeout** is 15 min (Lambda hard cap). Failures land in the DLQ after one retry.
+- **Worker `/tmp` cap** defaults to 30 GiB. Bump `worker_task_ephemeral_storage_gib` if users upload very large Discord exports (Fargate ceiling is 200 GiB).
+- **Worker has no time cap.** Heavy packages just take as long as they need; failures show up as ERRORED package rows + a Discord webhook from `process_package`, not in the DLQ.
+- **Forwarder DLQ.** If the forwarder Lambda itself fails to launch a Fargate task twice (capacity / IAM / network), the SQS message lands in the DLQ — alarmed via Discord through `monitoring.tf`.
 - **fck-nat is a single instance.** Switch to a managed NAT Gateway if you need the extra availability — at the cost of a much higher fixed monthly bill.
 
 ## API Documentation

infra/terraform/.terraform.lock.hcl

@@ -1,15 +1,17 @@
-# Solves the first-apply chicken-and-egg between ECR and Lambda:
-# CreateFunction (image_uri = ecr_repo:bootstrap) requires the image to exist,
-# but the image obviously doesn't exist until something pushes it.
+# Solves the first-apply chicken-and-egg between ECR and Lambda/ECS:
+# CreateFunction (image_uri = ecr_repo:bootstrap) and ECS task definitions
+# require the image to exist, but the image obviously doesn't exist until
+# something pushes it.
 #
-# This null_resource runs on first apply, pulls the public AWS Lambda Python
-# base image, and re-pushes it to our ECR with the :bootstrap tag. The Lambda
-# functions depend on it, so apply ordering becomes:
-#   ECR repo → null_resource (push placeholder) → Lambdas
+# These null_resources run on first apply, pull the relevant public base
+# image, and re-push it to our ECR with the :bootstrap tag. Lambda + ECS
+# resources depend on it, so apply ordering becomes:
+#   ECR repo → null_resource (push placeholder) → consumer
 #
-# After first apply, CI takes over: every push to main builds the real image
-# and calls update-function-code, which Terraform ignores via the
-# lifecycle.ignore_changes = [image_uri] block on the Lambda resources.
+# After first apply, CI takes over: every push to main builds the real
+# image and updates the consumer (Lambda update-function-code, ECS
+# register-task-definition). The lifecycle.ignore_changes blocks on
+# image_uri / container_definitions keep TF out of CI's way.
 #
 # Requires `docker` and `aws` CLI on the apply host.
 
@@ -27,9 +29,6 @@ resource "null_resource" "lambda_bootstrap_image" {
       region="${var.region}"
       repo_name="${aws_ecr_repository.lambda.name}"
 
-      # Idempotent: skip if a :bootstrap image already exists (e.g. after CI
-      # has run and the placeholder was overwritten with a real image, or after
-      # a re-apply that re-creates this null_resource for any reason).
       if aws ecr describe-images \
             --repository-name "$${repo_name}" \
             --image-ids imageTag=bootstrap \
@@ -41,8 +40,6 @@ resource "null_resource" "lambda_bootstrap_image" {
       aws ecr get-login-password --region "$${region}" \
         | docker login --username AWS --password-stdin "$${registry}"
 
-      # The placeholder just needs to satisfy CreateFunction's image-exists
-      # check; it is replaced on the first CI deploy. Pin a non-deprecated tag.
       placeholder="public.ecr.aws/lambda/python:3.13"
       docker pull --platform linux/amd64 "$${placeholder}"
       docker tag  "$${placeholder}" "$${repo_url}:bootstrap"
@@ -52,3 +49,39 @@ resource "null_resource" "lambda_bootstrap_image" {
 
   depends_on = [aws_ecr_repository.lambda]
 }
+
+resource "null_resource" "worker_bootstrap_image" {
+  triggers = {
+    repo_url = aws_ecr_repository.worker.repository_url
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
+    command     = <<-EOT
+      set -euo pipefail
+      repo_url="${aws_ecr_repository.worker.repository_url}"
+      registry="$${repo_url%%/*}"
+      region="${var.region}"
+      repo_name="${aws_ecr_repository.worker.name}"
+
+      if aws ecr describe-images \
+            --repository-name "$${repo_name}" \
+            --image-ids imageTag=bootstrap \
+            --region "$${region}" >/dev/null 2>&1; then
+        echo "bootstrap tag already present in $${repo_name}, skipping"
+        exit 0
+      fi
+
+      aws ecr get-login-password --region "$${region}" \
+        | docker login --username AWS --password-stdin "$${registry}"
+
+      # Plain Python base — what the worker image actually uses.
+      placeholder="public.ecr.aws/docker/library/python:3.10-slim"
+      docker pull --platform linux/amd64 "$${placeholder}"
+      docker tag  "$${placeholder}" "$${repo_url}:bootstrap"
+      docker push "$${repo_url}:bootstrap"
+    EOT
+  }
+
+  depends_on = [aws_ecr_repository.worker]
+}

infra/terraform/bootstrap.tf

@@ -1,5 +1,7 @@
-# One repo holds the image used by both Lambdas. The two functions select
-# their handler via ImageConfig.Command.
+# Two repos: one for the Lambda image (API + forwarder), one for the
+# Fargate worker image. They have different base images (Lambda runtime vs
+# plain Python) so they can't share.
+
 resource "aws_ecr_repository" "lambda" {
   name                 = "${local.name}-lambda"
   image_tag_mutability = "MUTABLE"
@@ -9,9 +11,17 @@ resource "aws_ecr_repository" "lambda" {
   }
 }
 
-resource "aws_ecr_lifecycle_policy" "lambda" {
-  repository = aws_ecr_repository.lambda.name
-  policy = jsonencode({
+resource "aws_ecr_repository" "worker" {
+  name                 = "${local.name}-worker"
+  image_tag_mutability = "MUTABLE"
+
+  image_scanning_configuration {
+    scan_on_push = true
+  }
+}
+
+locals {
+  ecr_lifecycle_policy = jsonencode({
     rules = [
       {
         rulePriority = 1
@@ -37,3 +47,13 @@ resource "aws_ecr_lifecycle_policy" "lambda" {
     ]
   })
 }
+
+resource "aws_ecr_lifecycle_policy" "lambda" {
+  repository = aws_ecr_repository.lambda.name
+  policy     = local.ecr_lifecycle_policy
+}
+
+resource "aws_ecr_lifecycle_policy" "worker" {
+  repository = aws_ecr_repository.worker.name
+  policy     = local.ecr_lifecycle_policy
+}