feat: deploy API + workers to AWS Lambda via Terraform

.env.example

@@ -4,17 +4,13 @@ POSTGRES_PORT="6641"
 POSTGRES_DB="dumpus"
 POSTGRES_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASS}@localhost:${POSTGRES_PORT}/${POSTGRES_DB}"
 
-RABBITMQ_USER="dumpus"
-RABBITMQ_PASS="CHANGE_ME"
-RABBITMQ_PORT="5672"
-RABBITMQ_URL="amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@localhost:${RABBITMQ_PORT}/"
-
 API_PORT="5000"
 DL_ZIP_WHITELISTED_DOMAINS=""
 DL_ZIP_TMP_PATH="./tmp"
 
-CELERY_QUEUE="regular_process"
-CELERY_HOSTNAME="regular_process"
+# `sync` runs the worker inline (default for local dev).
+# `sqs` (production) sends to AWS SQS — also requires SQS_QUEUE_URL.
+QUEUE_BACKEND="sync"
 
 DISWHO_JWT_SECRET=""
 DISWHO_BASE_URL=""

.github/workflows/deploy.yml

@@ -0,0 +1,102 @@
+name: deploy
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - "docs/**"
+      - "infra/**"
+      - "**.md"
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Override image tag (defaults to git sha)"
+        required: false
+
+permissions:
+  id-token: write   # GitHub OIDC
+  contents: read
+
+env:
+  AWS_REGION: eu-west-1
+  ECR_REPO: dumpus-prod-lambda
+  API_FUNCTION: dumpus-prod-api
+  WORKER_FUNCTION: dumpus-prod-worker
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Resolve image tag
+        id: meta
+        run: |
+          if [ -n "${{ inputs.image_tag }}" ]; then
+            tag="${{ inputs.image_tag }}"
+          else
+            tag="${GITHUB_SHA::12}"
+          fi
+          echo "image_tag=$tag" >> "$GITHUB_OUTPUT"
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to ECR
+        id: ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build & push Lambda image
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: Dockerfile.lambda
+          push: true
+          # Lambda only runs amd64 unless the function is configured for arm64.
+          # Keep amd64 unless we explicitly switch the function architecture.
+          platforms: linux/amd64
+          provenance: false
+          tags: |
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }}
+            ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:latest
+          cache-from: type=gha,scope=lambda
+          cache-to: type=gha,scope=lambda,mode=max
+
+      - name: Update Lambda functions
+        env:
+          IMAGE_URI: ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }}
+        run: |
+          set -euo pipefail
+          for fn in "${API_FUNCTION}" "${WORKER_FUNCTION}"; do
+            echo "::group::Updating $fn"
+            aws lambda update-function-code \
+              --function-name "$fn" \
+              --image-uri "$IMAGE_URI" \
+              --publish \
+              > /dev/null
+            aws lambda wait function-updated --function-name "$fn"
+            echo "::endgroup::"
+          done
+
+      - name: Smoke test API
+        run: |
+          set -euo pipefail
+          for i in 1 2 3 4 5; do
+            code=$(curl -s -o /dev/null -w "%{http_code}" https://api.dumpus.app/health || echo "000")
+            if [ "$code" = "200" ]; then
+              echo "API healthy"
+              exit 0
+            fi
+            echo "attempt $i: $code, retrying..."
+            sleep 5
+          done
+          echo "API never returned 200" >&2
+          exit 1

.github/workflows/infra.yml

@@ -0,0 +1,66 @@
+name: infra
+
+on:
+  workflow_dispatch:
+    inputs:
+      action:
+        type: choice
+        description: "What to do"
+        options: [plan, apply]
+        default: plan
+
+permissions:
+  contents: read
+
+jobs:
+  infra:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: infra/terraform
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up OpenTofu
+        uses: opentofu/setup-opentofu@v1
+        with:
+          tofu_version: 1.8.5
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          aws-access-key-id: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+
+      - name: tofu init
+        run: tofu init
+
+      - name: tofu plan
+        if: inputs.action == 'plan'
+        env:
+          TF_VAR_diswho_jwt_secret: ${{ secrets.DISWHO_JWT_SECRET }}
+          TF_VAR_wh_url: ${{ secrets.WH_URL }}
+        run: tofu plan -var-file=terraform.ci.tfvars -no-color | tee /tmp/plan.txt
+
+      - name: tofu apply
+        if: inputs.action == 'apply'
+        env:
+          TF_VAR_diswho_jwt_secret: ${{ secrets.DISWHO_JWT_SECRET }}
+          TF_VAR_wh_url: ${{ secrets.WH_URL }}
+        run: tofu apply -auto-approve -var-file=terraform.ci.tfvars
+
+      - name: Outputs
+        if: inputs.action == 'apply'
+        run: |
+          {
+            echo "## Terraform outputs"
+            echo
+            echo '```'
+            tofu output
+            echo '```'
+            echo
+            echo "**Next**: copy \`github_deploy_role_arn\` into the \`AWS_DEPLOY_ROLE_ARN\` repo secret so the deploy workflow can use OIDC."
+          } >> "$GITHUB_STEP_SUMMARY"

.gitignore

@@ -11,10 +11,15 @@ node_test
 
 # Env
 .env
+.env.*
+!.env.example
 
 # Data
 scripts/res.txt
 *.db
 
 # JetBrains IDE
-.idea
+.idea
+
+# Claude Code
+.claude/

Dockerfile.api

@@ -1,18 +1,8 @@
-# Last stable Python version
-FROM python:3.10-slim-buster
+FROM python:3.10-slim-bookworm
 
 # Create application directory and move there
 WORKDIR /app
 
-# Install nltk
-RUN pip install --no-cache-dir --prefer-binary nltk
-
-# Copy the script that downloads the vader lexicon module
-COPY scripts/download-ntk.py ./
-
-# Execute that script
-RUN python download-ntk.py
-
 # Copy requirements file from the host to the container
 COPY requirements.txt ./
 

Dockerfile.lambda

@@ -0,0 +1,15 @@
+# Single image used by both Lambda functions (api + worker). The active
+# handler is selected per-function via the Lambda ImageConfig.Command.
+FROM public.ecr.aws/lambda/python:3.10
+
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Flat layout: src/* sits at the task root so existing top-level imports
+# (`from tasks import ...`) keep working.
+COPY src/ ${LAMBDA_TASK_ROOT}/
+
+# Default to the API handler. Override per function in Terraform/Lambda config.
+CMD [ "lambda_handlers.api.handler" ]

Makefile

@@ -14,16 +14,12 @@ install:
 	uv venv --python 3.10
 	. .venv/bin/activate
 	uv pip install -r requirements.txt
-	uv run ./scripts/download-ntk.py
 
 dev:
 	. .venv/bin/activate
 	bash -c " \
 		trap 'docker compose down' EXIT; \
-		docker compose up -d broker db && \
+		docker compose up -d db && \
 		cd src && \
-		tee \
-			>(docker compose logs -f) \
-			>(celery --app tasks worker --loglevel=info --queues=${CELERY_QUEUE} --hostname=${CELERY_HOSTNAME}@%h --concurrency=1) \
-			>(waitress-serve --port=${API_PORT} app:app) \
-	"
+		QUEUE_BACKEND=sync waitress-serve --port=${API_PORT} app:app \
+	"