diff --git a/.env.example b/.env.example index 4e767ff..ecf037e 100644 --- a/.env.example +++ b/.env.example @@ -4,17 +4,13 @@ POSTGRES_PORT="6641" POSTGRES_DB="dumpus" POSTGRES_URL="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASS}@localhost:${POSTGRES_PORT}/${POSTGRES_DB}" -RABBITMQ_USER="dumpus" -RABBITMQ_PASS="CHANGE_ME" -RABBITMQ_PORT="5672" -RABBITMQ_URL="amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@localhost:${RABBITMQ_PORT}/" - API_PORT="5000" DL_ZIP_WHITELISTED_DOMAINS="" DL_ZIP_TMP_PATH="./tmp" -CELERY_QUEUE="regular_process" -CELERY_HOSTNAME="regular_process" +# `sync` runs the worker inline (default for local dev). +# `sqs` (production) sends to AWS SQS — also requires SQS_QUEUE_URL. +QUEUE_BACKEND="sync" DISWHO_JWT_SECRET="" DISWHO_BASE_URL="" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..3975988 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,102 @@ +name: deploy + +on: + push: + branches: [main] + paths-ignore: + - "docs/**" + - "infra/**" + - "**.md" + workflow_dispatch: + inputs: + image_tag: + description: "Override image tag (defaults to git sha)" + required: false + +permissions: + id-token: write # GitHub OIDC + contents: read + +env: + AWS_REGION: eu-west-1 + ECR_REPO: dumpus-prod-lambda + API_FUNCTION: dumpus-prod-api + WORKER_FUNCTION: dumpus-prod-worker + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Resolve image tag + id: meta + run: | + if [ -n "${{ inputs.image_tag }}" ]; then + tag="${{ inputs.image_tag }}" + else + tag="${GITHUB_SHA::12}" + fi + echo "image_tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to ECR + id: ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set up Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build & push Lambda image + id: build + uses: docker/build-push-action@v7 + with: + context: . + file: Dockerfile.lambda + push: true + # Lambda only runs amd64 unless the function is configured for arm64. + # Keep amd64 unless we explicitly switch the function architecture. + platforms: linux/amd64 + provenance: false + tags: | + ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }} + ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:latest + cache-from: type=gha,scope=lambda + cache-to: type=gha,scope=lambda,mode=max + + - name: Update Lambda functions + env: + IMAGE_URI: ${{ steps.ecr.outputs.registry }}/${{ env.ECR_REPO }}:${{ steps.meta.outputs.image_tag }} + run: | + set -euo pipefail + for fn in "${API_FUNCTION}" "${WORKER_FUNCTION}"; do + echo "::group::Updating $fn" + aws lambda update-function-code \ + --function-name "$fn" \ + --image-uri "$IMAGE_URI" \ + --publish \ + > /dev/null + aws lambda wait function-updated --function-name "$fn" + echo "::endgroup::" + done + + - name: Smoke test API + run: | + set -euo pipefail + for i in 1 2 3 4 5; do + code=$(curl -s -o /dev/null -w "%{http_code}" https://api.dumpus.app/health || echo "000") + if [ "$code" = "200" ]; then + echo "API healthy" + exit 0 + fi + echo "attempt $i: $code, retrying..." + sleep 5 + done + echo "API never returned 200" >&2 + exit 1 diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml new file mode 100644 index 0000000..9bbb740 --- /dev/null +++ b/.github/workflows/infra.yml @@ -0,0 +1,66 @@ +name: infra + +on: + workflow_dispatch: + inputs: + action: + type: choice + description: "What to do" + options: [plan, apply] + default: plan + +permissions: + contents: read + +jobs: + infra: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: infra/terraform + + steps: + - uses: actions/checkout@v6 + + - name: Set up OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: 1.8.5 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + aws-access-key-id: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + + - name: tofu init + run: tofu init + + - name: tofu plan + if: inputs.action == 'plan' + env: + TF_VAR_diswho_jwt_secret: ${{ secrets.DISWHO_JWT_SECRET }} + TF_VAR_wh_url: ${{ secrets.WH_URL }} + run: tofu plan -var-file=terraform.ci.tfvars -no-color | tee /tmp/plan.txt + + - name: tofu apply + if: inputs.action == 'apply' + env: + TF_VAR_diswho_jwt_secret: ${{ secrets.DISWHO_JWT_SECRET }} + TF_VAR_wh_url: ${{ secrets.WH_URL }} + run: tofu apply -auto-approve -var-file=terraform.ci.tfvars + + - name: Outputs + if: inputs.action == 'apply' + run: | + { + echo "## Terraform outputs" + echo + echo '```' + tofu output + echo '```' + echo + echo "**Next**: copy \`github_deploy_role_arn\` into the \`AWS_DEPLOY_ROLE_ARN\` repo secret so the deploy workflow can use OIDC." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 13aa4c6..b7efa53 100644 --- a/.gitignore +++ b/.gitignore @@ -11,10 +11,15 @@ node_test # Env .env +.env.* +!.env.example # Data scripts/res.txt *.db # JetBrains IDE -.idea \ No newline at end of file +.idea + +# Claude Code +.claude/ \ No newline at end of file diff --git a/Dockerfile.api b/Dockerfile.api index dcf31ba..8e2a4fe 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,18 +1,8 @@ -# Last stable Python version -FROM python:3.10-slim-buster +FROM python:3.10-slim-bookworm # Create application directory and move there WORKDIR /app -# Install nltk -RUN pip install --no-cache-dir --prefer-binary nltk - -# Copy the script that downloads the vader lexicon module -COPY scripts/download-ntk.py ./ - -# Execute that script -RUN python download-ntk.py - # Copy requirements file from the host to the container COPY requirements.txt ./ diff --git a/Dockerfile.flower b/Dockerfile.flower deleted file mode 100644 index dada7e6..0000000 --- a/Dockerfile.flower +++ /dev/null @@ -1,26 +0,0 @@ -# Last stable Python version -FROM python:3.10-slim-buster - -# Install curl -RUN apt-get update && apt-get install -y curl - -# Create application directory and move there -WORKDIR /app - -# Copy requirements file from the host to the container -COPY requirements.txt ./ - -# Install these requirements -RUN pip install --no-cache-dir -r requirements.txt - -# Copy the rest of the application files -COPY . . - -# Expose UI port -EXPOSE 5566 - -WORKDIR /app/src - -# Start the Celery flower -# celery -A tasks flower --port=5566 -CMD ["celery", "-A", "tasks", "flower", "--port=5566"] diff --git a/Dockerfile.lambda b/Dockerfile.lambda new file mode 100644 index 0000000..c71d795 --- /dev/null +++ b/Dockerfile.lambda @@ -0,0 +1,15 @@ +# Single image used by both Lambda functions (api + worker). The active +# handler is selected per-function via the Lambda ImageConfig.Command. +FROM public.ecr.aws/lambda/python:3.10 + +WORKDIR ${LAMBDA_TASK_ROOT} + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +# Flat layout: src/* sits at the task root so existing top-level imports +# (`from tasks import ...`) keep working. +COPY src/ ${LAMBDA_TASK_ROOT}/ + +# Default to the API handler. Override per function in Terraform/Lambda config. +CMD [ "lambda_handlers.api.handler" ] diff --git a/Dockerfile.worker b/Dockerfile.worker deleted file mode 100644 index d399904..0000000 --- a/Dockerfile.worker +++ /dev/null @@ -1,31 +0,0 @@ -# Last stable Python version -FROM python:3.10-slim-buster - -# Install curl -RUN apt-get update && apt-get install -y curl - -# Create application directory and move there -WORKDIR /app - -# Install nltk -RUN pip install --no-cache-dir --prefer-binary nltk - -# Copy the script that downloads the vader lexicon module -COPY scripts/download-ntk.py ./ - -# Execute that script -RUN python download-ntk.py - -# Copy requirements file from the host to the container -COPY requirements.txt ./ - -# Install these requirements -RUN pip install --no-cache-dir -r requirements.txt - -# Copy the rest of the application files -COPY . . - -WORKDIR /app/src - -# Start the Celery worker -CMD celery --app tasks worker --loglevel=info --queues=$CELERY_QUEUE --hostname=$CELERY_HOSTNAME@%h --concurrency=1 diff --git a/Makefile b/Makefile index bb1d25a..454dd47 100644 --- a/Makefile +++ b/Makefile @@ -14,16 +14,12 @@ install: uv venv --python 3.10 . .venv/bin/activate uv pip install -r requirements.txt - uv run ./scripts/download-ntk.py dev: . .venv/bin/activate bash -c " \ trap 'docker compose down' EXIT; \ - docker compose up -d broker db && \ + docker compose up -d db && \ cd src && \ - tee \ - >(docker compose logs -f) \ - >(celery --app tasks worker --loglevel=info --queues=${CELERY_QUEUE} --hostname=${CELERY_HOSTNAME}@%h --concurrency=1) \ - >(waitress-serve --port=${API_PORT} app:app) \ - " \ No newline at end of file + QUEUE_BACKEND=sync waitress-serve --port=${API_PORT} app:app \ + " diff --git a/README.md b/README.md index 41fa98b..84f2e76 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ API to extract statistics from the Discord Data Packages (GDPR packages). This A * [Architecture Documentation](#architecture-documentation) * [Self-hosting](#self-hosting) +* [Deploy to AWS](#deploy-to-aws) * [API Documentation](#api-documentation) * [Troubleshooting](#troubleshooting) @@ -32,18 +33,59 @@ Thus: Anyone can host their own Dumpus instance. The official Dumpus client can then be configured to use it. +The worker no longer runs as a separate Celery process — set `QUEUE_BACKEND=sync` and the API processes packages inline on the request thread (good enough at small volume), or set `QUEUE_BACKEND=sqs` to dispatch to AWS SQS (used by the Lambda deployment). + * clone -* you can use Docker (easy way) -* or you can install everything by yourself: +* easy way: `cp .env.example .env`, fill it in, then `make up` +* manual: - install requirements with pip - - start a RabbitMQ server (or redis) - start a PostgreSQL server - - fill the .env file with your Redis and PostgreSQL creds - - start the API using `waitress-serve --port=5000 app:app` - - start one worker using `celery --app tasks worker --loglevel=info --queues=regular_process --hostname=regular_process@%h --concurrency=1` + - fill the .env file with your PostgreSQL creds + - start the API: `QUEUE_BACKEND=sync waitress-serve --port=5000 app:app` By default, Dumpus API will only treat zip files sent from `https://discord.click`. You can specify a `DL_ZIP_WHITELISTED_DOMAINS` environment variable to add other allowed domains. +## Deploy to AWS + +A Terraform stack under `infra/terraform/` provisions a serverless deployment of the API on AWS: + +| Component | AWS service | +| ---------------- | ------------------------ | +| API | Lambda (container image) behind API Gateway HTTP API | +| Worker | Lambda triggered by SQS | +| Database | RDS Postgres in private subnets | +| Outbound NAT | fck-nat instance (NAT Gateway replacement) | +| Secrets | Secrets Manager + Lambda env | +| TLS / DNS | ACM cert + Route53 alias to API Gateway | +| CI | GitHub OIDC role; build → ECR → `update-function-code` | + +### Bootstrap + +1. Create a public Route53 hosted zone for your domain and point your registrar's nameservers at it. +2. `cp infra/terraform/terraform.tfvars.example infra/terraform/terraform.tfvars` and fill in `discord_secret`, `domain_name`, `github_repository`, region, etc. +3. `cd infra/terraform && terraform init && terraform apply`. + This single apply does everything: a `null_resource` pushes a placeholder image (the public AWS Lambda Python base) into ECR with the `:bootstrap` tag, then the Lambda functions are created against that placeholder. Requires `docker` and `aws` CLI on the apply host. +4. Set the GitHub repo secret `AWS_DEPLOY_ROLE_ARN` from `terraform output -raw github_deploy_role_arn`. From here on, every push to `main` builds the real image in CI and rolls both Lambdas — no more local builds needed. + +### Day-to-day deploys + +Push to `main` → `.github/workflows/deploy.yml` builds the container, pushes to ECR tagged with the git SHA, and rolls both Lambdas via OIDC. No long-lived AWS keys in GitHub. + +### Operations + +```bash +aws logs tail /aws/lambda/--api --follow +aws logs tail /aws/lambda/--worker --follow +aws sqs receive-message --queue-url "$(terraform output -raw sqs_dlq_url)" +``` + +Things to keep in mind: + +- **API cold start** is a few seconds while pandas imports. Invisible on the async submit/poll flow; use provisioned concurrency if a sync endpoint must be sub-second. +- **Worker `/tmp` cap** defaults to 5GB (max 10GB). Bump `worker_ephemeral_storage_mb` if users upload very large Discord exports. +- **Worker timeout** is 15 min (Lambda hard cap). Failures land in the DLQ after one retry. +- **fck-nat is a single instance.** Switch to a managed NAT Gateway if you need the extra availability — at the cost of a much higher fixed monthly bill. + ## API Documentation One header is required for all the requests except the `POST /process` one: @@ -167,12 +209,5 @@ Current error message codes: ## Troubleshooting -* Server does not respond after `POST /process`. Try to remove this property from the celeryconfig.py file. -``` -broker_use_ssl={ - 'ssl_cert_reqs': None -} -``` - -* API server is crashing and say that Postgres is not supported. +* API server is crashing and says that Postgres is not supported. Make sure that your PostgreSQL server URL starts with **postgresql://** and not **postgres://**, which is no longer supported by SQLAlchemy. diff --git a/captain.flower b/captain.flower deleted file mode 100644 index 5c98827..0000000 --- a/captain.flower +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schemaVersion": 2, - "dockerfilePath": "Dockerfile.flower" -} \ No newline at end of file diff --git a/captain.worker b/captain.worker deleted file mode 100644 index a51817a..0000000 --- a/captain.worker +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schemaVersion": 2, - "dockerfilePath": "Dockerfile.worker" -} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 387fdb7..3189bc6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,4 @@ services: - broker: - container_name: dumpus-broker - image: rabbitmq:latest - ports: - - "${RABBITMQ_PORT}:5672" - - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "--quiet", "ping"] - timeout: 10s - retries: 5 - start_period: 10s - - environment: - RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER} - RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASS} - - restart: on-failure - db: container_name: dumpus-db image: postgres:17-alpine @@ -38,35 +20,6 @@ services: volumes: - postgres-data:/var/lib/postgresql/data - worker: - container_name: dumpus-worker - build: - dockerfile: Dockerfile.worker - - healthcheck: - test: - [ - "CMD-SHELL", - "celery -A tasks inspect ping -d regular_process@$$HOSTNAME", - ] - interval: 2s - timeout: 10s - retries: 5 - - depends_on: - broker: - condition: service_healthy - restart: true - db: - condition: service_healthy - restart: true - - environment: - POSTGRES_URL: "postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASS}@db:5432/dumpus" - RABBITMQ_URL: "amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@broker:5672/" - CELERY_HOSTNAME: "${CELERY_HOSTNAME}" - CELERY_QUEUE: "${CELERY_QUEUE}" - api: container_name: dumpus-api build: @@ -75,26 +28,20 @@ services: ports: - "${API_PORT}:5000" depends_on: - worker: + db: condition: service_healthy restart: true + # Local dev runs the worker in-process: enqueue_package() calls process_package() + # synchronously. In production, the API Lambda enqueues to SQS instead. environment: + QUEUE_BACKEND: sync POSTGRES_URL: "postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASS}@db:5432/dumpus" - RABBITMQ_URL: "amqp://${RABBITMQ_USER}:${RABBITMQ_PASS}@broker:5672/" - - # flower: - # container_name: flower - # build: - # dockerfile: Dockerfile.flower - - # ports: - # - "5566:5566" - - # depends_on: - # worker: - # condition: service_healthy - # restart: true + DL_ZIP_TMP_PATH: ${DL_ZIP_TMP_PATH} + DL_ZIP_WHITELISTED_DOMAINS: ${DL_ZIP_WHITELISTED_DOMAINS} + DISWHO_BASE_URL: ${DISWHO_BASE_URL} + DISWHO_JWT_SECRET: ${DISWHO_JWT_SECRET} + DISCORD_SECRET: ${DISCORD_SECRET} volumes: postgres-data: diff --git a/infra/terraform/.gitignore b/infra/terraform/.gitignore new file mode 100644 index 0000000..6db0fdd --- /dev/null +++ b/infra/terraform/.gitignore @@ -0,0 +1,12 @@ +.terraform/ +*.tfstate +*.tfstate.* +*.tfvars +!*.tfvars.example +!terraform.ci.tfvars +crash.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json +# .terraform.lock.hcl IS committed — it pins provider versions for reproducibility. diff --git a/infra/terraform/.terraform.lock.hcl b/infra/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..2b6459b --- /dev/null +++ b/infra/terraform/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "6.43.0" + constraints = "~> 6.0" + hashes = [ + "h1:WRONI5OW8FuSWm2YXR8K4I6JtBvuJG9dGokcNAYRkUw=", + "zh:108a58036307f9f0687d2583f86936eb75401e43d358ea62f9702acf5f9d70b2", + "zh:373cb307d87cab9806f8cd71388d7ba451d7bb310a9e2807c7afb80cee6841a2", + "zh:42a59943c3f46e3e17ab707df0fcf6313916389383866d3017a467ad37993924", + "zh:5c3b005efb2ec73833cbb80cd9d1bd2fae27586c510f4c58a7ed28d6c4f1df8f", + "zh:612ee0001c46dae19255252c3a9344bc0fcdda42568508dda558079ed8cf437f", + "zh:625d1b0c8e4dc0bb43583c4ad0e7a30c030e3e67d919b6c5e8247707b7ca1e6b", + "zh:92ecce00bdf17b63fcf31fb8eb7679cfbec0c741097554055f810ee879a96094", + "zh:b1fa126cb4f52c1fc00beda5c3fa34fac41a95f5a2ad3a14751e335bc93ee7ab", + "zh:b54a80475ac532f44f7c8cb900c48c01780b50f5dae0fe852175f3331ca9371b", + "zh:bb77f0c00778d5ac3257ded46c1034df656bdd67f3cf29e74e68da376fea1ec3", + "zh:bedcc72914998315e8faac3997b5eb6ac49269a14e557efd5916c1946e734179", + "zh:cbf429abd34c247a57ef1b703d059fea30ecf5463ee0d20a2850bd25adb22d5a", + "zh:cfbc5decdfce32cde83de9f7e10e30b30297246611c69a2243d88a6421d2dd7a", + "zh:d656ba4ddd17f108397d7dfd28670a7d70a9c4d46229621748e8755783e391fc", + "zh:f304352bb8936b9d7dc9f57273c50db72768004ed4789b5712e20b96c3342478", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.8.1" + constraints = "~> 3.6" + hashes = [ + "h1:LsYuJLZcYl1RiH7Hd3w90Ra5+k5cNqfdRUQXItkTI8Y=", + "zh:25c458c7c676f15705e872202dad7dcd0982e4a48e7ea1800afa5fc64e77f4c8", + "zh:2edeaf6f1b20435b2f81855ad98a2e70956d473be9e52a5fdf57ccd0098ba476", + "zh:44becb9d5f75d55e36dfed0c5beabaf4c92e0a2bc61a3814d698271c646d48e7", + "zh:7699032612c3b16cc69928add8973de47b10ce81b1141f30644a0e8a895b5cd3", + "zh:86d07aa98d17703de9fbf402c89590dc1e01dbe5671dd6bc5e487eb8fe87eee0", + "zh:8c411c77b8390a49a8a1bc9f176529e6b32369dd33a723606c8533e5ca4d68c1", + "zh:a5ecc8255a612652a56b28149994985e2c4dc046e5d34d416d47fa7767f5c28f", + "zh:aea3fe1a5669b932eda9c5c72e5f327db8da707fe514aaca0d0ef60cb24892f9", + "zh:f56e26e6977f755d7ae56fa6320af96ecf4bb09580d47cb481efbf27f1c5afff", + ] +} diff --git a/infra/terraform/apigw.tf b/infra/terraform/apigw.tf new file mode 100644 index 0000000..3f95a7a --- /dev/null +++ b/infra/terraform/apigw.tf @@ -0,0 +1,77 @@ +resource "aws_apigatewayv2_api" "api" { + name = "${local.name}-api" + protocol_type = "HTTP" + + cors_configuration { + allow_origins = ["*"] + allow_methods = ["GET", "POST", "DELETE", "OPTIONS"] + allow_headers = ["authorization", "content-type"] + max_age = 300 + } +} + +resource "aws_apigatewayv2_integration" "api" { + api_id = aws_apigatewayv2_api.api.id + integration_type = "AWS_PROXY" + integration_uri = aws_lambda_function.api.invoke_arn + payload_format_version = "2.0" + timeout_milliseconds = 30000 +} + +resource "aws_apigatewayv2_route" "default" { + api_id = aws_apigatewayv2_api.api.id + route_key = "$default" + target = "integrations/${aws_apigatewayv2_integration.api.id}" +} + +resource "aws_apigatewayv2_stage" "default" { + api_id = aws_apigatewayv2_api.api.id + name = "$default" + auto_deploy = true + + access_log_settings { + destination_arn = aws_cloudwatch_log_group.apigw.arn + format = jsonencode({ + requestId = "$context.requestId" + ip = "$context.identity.sourceIp" + requestTime = "$context.requestTime" + method = "$context.httpMethod" + routeKey = "$context.routeKey" + status = "$context.status" + protocol = "$context.protocol" + responseLength = "$context.responseLength" + integrationLatency = "$context.integrationLatency" + }) + } + + default_route_settings { + throttling_burst_limit = 50 + throttling_rate_limit = 25 + } +} + +resource "aws_lambda_permission" "apigw_invoke_api" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.api.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_apigatewayv2_api.api.execution_arn}/*/*" +} + +# --- Custom domain for api.dumpus.app --- + +resource "aws_apigatewayv2_domain_name" "api" { + domain_name = local.api_fqdn + + domain_name_configuration { + certificate_arn = aws_acm_certificate_validation.api.certificate_arn + endpoint_type = "REGIONAL" + security_policy = "TLS_1_2" + } +} + +resource "aws_apigatewayv2_api_mapping" "api" { + api_id = aws_apigatewayv2_api.api.id + domain_name = aws_apigatewayv2_domain_name.api.id + stage = aws_apigatewayv2_stage.default.id +} diff --git a/infra/terraform/backend.tf b/infra/terraform/backend.tf new file mode 100644 index 0000000..78c61e2 --- /dev/null +++ b/infra/terraform/backend.tf @@ -0,0 +1,9 @@ +terraform { + backend "s3" { + bucket = "dumpus-prod-tfstate" + key = "api/prod/terraform.tfstate" + region = "eu-west-1" + dynamodb_table = "dumpus-prod-tfstate-lock" + encrypt = true + } +} diff --git a/infra/terraform/bootstrap.tf b/infra/terraform/bootstrap.tf new file mode 100644 index 0000000..b96cc55 --- /dev/null +++ b/infra/terraform/bootstrap.tf @@ -0,0 +1,54 @@ +# Solves the first-apply chicken-and-egg between ECR and Lambda: +# CreateFunction (image_uri = ecr_repo:bootstrap) requires the image to exist, +# but the image obviously doesn't exist until something pushes it. +# +# This null_resource runs on first apply, pulls the public AWS Lambda Python +# base image, and re-pushes it to our ECR with the :bootstrap tag. The Lambda +# functions depend on it, so apply ordering becomes: +# ECR repo → null_resource (push placeholder) → Lambdas +# +# After first apply, CI takes over: every push to main builds the real image +# and calls update-function-code, which Terraform ignores via the +# lifecycle.ignore_changes = [image_uri] block on the Lambda resources. +# +# Requires `docker` and `aws` CLI on the apply host. + +resource "null_resource" "lambda_bootstrap_image" { + triggers = { + repo_url = aws_ecr_repository.lambda.repository_url + } + + provisioner "local-exec" { + interpreter = ["bash", "-c"] + command = <<-EOT + set -euo pipefail + repo_url="${aws_ecr_repository.lambda.repository_url}" + registry="$${repo_url%%/*}" + region="${var.region}" + repo_name="${aws_ecr_repository.lambda.name}" + + # Idempotent: skip if a :bootstrap image already exists (e.g. after CI + # has run and the placeholder was overwritten with a real image, or after + # a re-apply that re-creates this null_resource for any reason). + if aws ecr describe-images \ + --repository-name "$${repo_name}" \ + --image-ids imageTag=bootstrap \ + --region "$${region}" >/dev/null 2>&1; then + echo "bootstrap tag already present in $${repo_name}, skipping" + exit 0 + fi + + aws ecr get-login-password --region "$${region}" \ + | docker login --username AWS --password-stdin "$${registry}" + + # The placeholder just needs to satisfy CreateFunction's image-exists + # check; it is replaced on the first CI deploy. Pin a non-deprecated tag. + placeholder="public.ecr.aws/lambda/python:3.13" + docker pull --platform linux/amd64 "$${placeholder}" + docker tag "$${placeholder}" "$${repo_url}:bootstrap" + docker push "$${repo_url}:bootstrap" + EOT + } + + depends_on = [aws_ecr_repository.lambda] +} diff --git a/infra/terraform/dns.tf b/infra/terraform/dns.tf new file mode 100644 index 0000000..35884df --- /dev/null +++ b/infra/terraform/dns.tf @@ -0,0 +1,48 @@ +data "aws_route53_zone" "apex" { + name = var.domain_name + private_zone = false +} + +resource "aws_acm_certificate" "api" { + domain_name = local.api_fqdn + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "acm_validation" { + for_each = { + for dvo in aws_acm_certificate.api.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + type = dvo.resource_record_type + record = dvo.resource_record_value + } + } + + zone_id = data.aws_route53_zone.apex.zone_id + name = each.value.name + type = each.value.type + ttl = 60 + records = [each.value.record] + allow_overwrite = true +} + +resource "aws_acm_certificate_validation" "api" { + certificate_arn = aws_acm_certificate.api.arn + validation_record_fqdns = [for r in aws_route53_record.acm_validation : r.fqdn] +} + +resource "aws_route53_record" "api" { + zone_id = data.aws_route53_zone.apex.zone_id + name = local.api_fqdn + type = "A" + allow_overwrite = true + + alias { + name = aws_apigatewayv2_domain_name.api.domain_name_configuration[0].target_domain_name + zone_id = aws_apigatewayv2_domain_name.api.domain_name_configuration[0].hosted_zone_id + evaluate_target_health = false + } +} diff --git a/infra/terraform/ecr.tf b/infra/terraform/ecr.tf new file mode 100644 index 0000000..c8eeca9 --- /dev/null +++ b/infra/terraform/ecr.tf @@ -0,0 +1,39 @@ +# One repo holds the image used by both Lambdas. The two functions select +# their handler via ImageConfig.Command. +resource "aws_ecr_repository" "lambda" { + name = "${local.name}-lambda" + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = true + } +} + +resource "aws_ecr_lifecycle_policy" "lambda" { + repository = aws_ecr_repository.lambda.name + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Expire untagged images after 7 days" + selection = { + tagStatus = "untagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = 7 + } + action = { type = "expire" } + }, + { + rulePriority = 2 + description = "Keep only the last 20 tagged images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 20 + } + action = { type = "expire" } + }, + ] + }) +} diff --git a/infra/terraform/github_oidc.tf b/infra/terraform/github_oidc.tf new file mode 100644 index 0000000..5e149b4 --- /dev/null +++ b/infra/terraform/github_oidc.tf @@ -0,0 +1,110 @@ +# GitHub Actions OIDC role — push images to ECR + roll new code into both Lambdas. +# Set var.github_repository to "" to skip provisioning. + +variable "github_repository" { + description = "Owner/repo allowed to assume the deploy role (e.g. Androz2091/dumpus-api). Empty = skip." + type = string + default = "" +} + +variable "github_deploy_branches" { + description = "Branches allowed to deploy" + type = list(string) + default = ["main"] +} + +locals { + github_oidc_enabled = var.github_repository != "" +} + +resource "aws_iam_openid_connect_provider" "github" { + count = local.github_oidc_enabled ? 1 : 0 + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = ["6938fd4d98bab03faadb97b34396831e3780aea1"] +} + +data "aws_iam_policy_document" "github_assume" { + count = local.github_oidc_enabled ? 1 : 0 + + statement { + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [aws_iam_openid_connect_provider.github[0].arn] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + + condition { + test = "StringLike" + variable = "token.actions.githubusercontent.com:sub" + values = [ + for branch in var.github_deploy_branches : + "repo:${var.github_repository}:ref:refs/heads/${branch}" + ] + } + } +} + +resource "aws_iam_role" "github_deploy" { + count = local.github_oidc_enabled ? 1 : 0 + name = "${local.name}-github-deploy" + assume_role_policy = data.aws_iam_policy_document.github_assume[0].json +} + +data "aws_iam_policy_document" "github_deploy" { + count = local.github_oidc_enabled ? 1 : 0 + + statement { + sid = "EcrAuth" + actions = ["ecr:GetAuthorizationToken"] + resources = ["*"] + } + + statement { + sid = "EcrPush" + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:CompleteLayerUpload", + "ecr:DescribeImages", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart", + ] + resources = [aws_ecr_repository.lambda.arn] + } + + statement { + sid = "LambdaUpdate" + actions = [ + "lambda:UpdateFunctionCode", + "lambda:GetFunction", + "lambda:PublishVersion", + "lambda:UpdateAlias", + "lambda:GetAlias", + ] + resources = [ + aws_lambda_function.api.arn, + aws_lambda_function.worker.arn, + ] + } +} + +resource "aws_iam_role_policy" "github_deploy" { + count = local.github_oidc_enabled ? 1 : 0 + name = "${local.name}-github-deploy" + role = aws_iam_role.github_deploy[0].id + policy = data.aws_iam_policy_document.github_deploy[0].json +} + +output "github_deploy_role_arn" { + description = "Set as the AWS_DEPLOY_ROLE_ARN secret in GitHub" + value = local.github_oidc_enabled ? aws_iam_role.github_deploy[0].arn : "" +} diff --git a/infra/terraform/iam.tf b/infra/terraform/iam.tf new file mode 100644 index 0000000..70422a9 --- /dev/null +++ b/infra/terraform/iam.tf @@ -0,0 +1,96 @@ +data "aws_iam_policy_document" "lambda_assume" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + } +} + +# --- API Lambda role --- + +resource "aws_iam_role" "api" { + name = "${local.name}-api-lambda" + assume_role_policy = data.aws_iam_policy_document.lambda_assume.json +} + +resource "aws_iam_role_policy_attachment" "api_basic" { + role = aws_iam_role.api.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_iam_role_policy_attachment" "api_vpc" { + role = aws_iam_role.api.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +data "aws_iam_policy_document" "api" { + statement { + sid = "EnqueuePackage" + actions = ["sqs:SendMessage"] + resources = [aws_sqs_queue.packages.arn] + } + + statement { + sid = "ReadAppSecrets" + actions = ["secretsmanager:GetSecretValue"] + resources = [ + aws_secretsmanager_secret.postgres_url.arn, + aws_secretsmanager_secret.diswho_jwt_secret.arn, + aws_secretsmanager_secret.wh_url.arn, + ] + } +} + +resource "aws_iam_role_policy" "api" { + name = "${local.name}-api" + role = aws_iam_role.api.id + policy = data.aws_iam_policy_document.api.json +} + +# --- Worker Lambda role --- + +resource "aws_iam_role" "worker" { + name = "${local.name}-worker-lambda" + assume_role_policy = data.aws_iam_policy_document.lambda_assume.json +} + +resource "aws_iam_role_policy_attachment" "worker_basic" { + role = aws_iam_role.worker.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" +} + +resource "aws_iam_role_policy_attachment" "worker_vpc" { + role = aws_iam_role.worker.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + +data "aws_iam_policy_document" "worker" { + statement { + sid = "ConsumeQueue" + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ChangeMessageVisibility", + ] + resources = [aws_sqs_queue.packages.arn] + } + + statement { + sid = "ReadAppSecrets" + actions = ["secretsmanager:GetSecretValue"] + resources = [ + aws_secretsmanager_secret.postgres_url.arn, + aws_secretsmanager_secret.diswho_jwt_secret.arn, + aws_secretsmanager_secret.wh_url.arn, + ] + } +} + +resource "aws_iam_role_policy" "worker" { + name = "${local.name}-worker" + role = aws_iam_role.worker.id + policy = data.aws_iam_policy_document.worker.json +} diff --git a/infra/terraform/lambda.tf b/infra/terraform/lambda.tf new file mode 100644 index 0000000..9cfa872 --- /dev/null +++ b/infra/terraform/lambda.tf @@ -0,0 +1,121 @@ +locals { + image_uri = "${aws_ecr_repository.lambda.repository_url}:${var.image_tag}" + + app_environment = { + DL_ZIP_TMP_PATH = "/tmp" + DL_ZIP_WHITELISTED_DOMAINS = var.dl_zip_whitelisted_domains + DISWHO_BASE_URL = var.diswho_base_url + QUEUE_BACKEND = "sqs" + SQS_QUEUE_URL = aws_sqs_queue.packages.url + + # Sensitive values are NOT in env. The Lambda's secrets_loader.py reads + # them from Secrets Manager at startup using the ARNs below. + SECRETS_ARN_MAP = jsonencode({ + POSTGRES_URL = aws_secretsmanager_secret.postgres_url.arn + DISWHO_JWT_SECRET = aws_secretsmanager_secret.diswho_jwt_secret.arn + WH_URL = aws_secretsmanager_secret.wh_url.arn + }) + } +} + +# --- API Lambda --- + +resource "aws_lambda_function" "api" { + function_name = "${local.name}-api" + role = aws_iam_role.api.arn + + package_type = "Image" + image_uri = local.image_uri + + image_config { + command = ["lambda_handlers.api.handler"] + } + + memory_size = var.api_lambda_memory + timeout = var.api_lambda_timeout + + vpc_config { + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.lambda.id] + } + + environment { + variables = local.app_environment + } + + # Deploys via CI bump image_uri without TF noticing — only the tag changes, + # the rest of the function spec is owned by Terraform. + lifecycle { + ignore_changes = [image_uri] + } + + depends_on = [ + aws_iam_role_policy_attachment.api_basic, + aws_iam_role_policy_attachment.api_vpc, + aws_cloudwatch_log_group.api, + null_resource.lambda_bootstrap_image, + ] +} + +# --- Worker Lambda --- + +resource "aws_lambda_function" "worker" { + function_name = "${local.name}-worker" + role = aws_iam_role.worker.arn + + package_type = "Image" + image_uri = local.image_uri + + image_config { + command = ["lambda_handlers.worker.handler"] + } + + memory_size = var.worker_lambda_memory + timeout = var.worker_lambda_timeout + + ephemeral_storage { + size = var.worker_ephemeral_storage_mb + } + + # null → use the account-level concurrency pool. New AWS accounts have a + # 10-execution ceiling and require ≥10 unreserved, so any reservation fails + # until you request a quota increase. Set the var > 0 once that's lifted. + reserved_concurrent_executions = var.worker_reserved_concurrency > 0 ? var.worker_reserved_concurrency : null + + vpc_config { + subnet_ids = aws_subnet.private[*].id + security_group_ids = [aws_security_group.lambda.id] + } + + environment { + variables = local.app_environment + } + + lifecycle { + ignore_changes = [image_uri] + } + + depends_on = [ + aws_iam_role_policy_attachment.worker_basic, + aws_iam_role_policy_attachment.worker_vpc, + aws_cloudwatch_log_group.worker, + null_resource.lambda_bootstrap_image, + ] +} + +# --- SQS → worker trigger --- + +resource "aws_lambda_event_source_mapping" "worker_sqs" { + event_source_arn = aws_sqs_queue.packages.arn + function_name = aws_lambda_function.worker.arn + + batch_size = 1 + maximum_batching_window_in_seconds = 0 + + function_response_types = ["ReportBatchItemFailures"] + + scaling_config { + # Match reserved concurrency so we don't burn through retries during a backlog. + maximum_concurrency = max(2, var.worker_reserved_concurrency) + } +} diff --git a/infra/terraform/logs.tf b/infra/terraform/logs.tf new file mode 100644 index 0000000..ebdcddc --- /dev/null +++ b/infra/terraform/logs.tf @@ -0,0 +1,14 @@ +resource "aws_cloudwatch_log_group" "api" { + name = "/aws/lambda/${local.name}-api" + retention_in_days = 30 +} + +resource "aws_cloudwatch_log_group" "worker" { + name = "/aws/lambda/${local.name}-worker" + retention_in_days = 30 +} + +resource "aws_cloudwatch_log_group" "apigw" { + name = "/aws/apigw/${local.name}" + retention_in_days = 30 +} diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf new file mode 100644 index 0000000..ca57b08 --- /dev/null +++ b/infra/terraform/main.tf @@ -0,0 +1,8 @@ +locals { + name = "${var.name_prefix}-${var.environment}" + api_fqdn = "${var.api_subdomain}.${var.domain_name}" +} + +data "aws_availability_zones" "available" { + state = "available" +} diff --git a/infra/terraform/network.tf b/infra/terraform/network.tf new file mode 100644 index 0000000..5c9a50d --- /dev/null +++ b/infra/terraform/network.tf @@ -0,0 +1,173 @@ +resource "aws_vpc" "main" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = "${local.name}-vpc" } +} + +resource "aws_internet_gateway" "main" { + vpc_id = aws_vpc.main.id + tags = { Name = "${local.name}-igw" } +} + +# Public subnets host the fck-nat instance. +resource "aws_subnet" "public" { + count = var.az_count + vpc_id = aws_vpc.main.id + cidr_block = cidrsubnet(var.vpc_cidr, 4, count.index) + availability_zone = data.aws_availability_zones.available.names[count.index] + map_public_ip_on_launch = true + + tags = { + Name = "${local.name}-public-${count.index}" + Tier = "public" + } +} + +# Private subnets host RDS and the Lambda ENIs. +resource "aws_subnet" "private" { + count = var.az_count + vpc_id = aws_vpc.main.id + cidr_block = cidrsubnet(var.vpc_cidr, 4, count.index + 8) + availability_zone = data.aws_availability_zones.available.names[count.index] + + tags = { + Name = "${local.name}-private-${count.index}" + Tier = "private" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.main.id + } + + tags = { Name = "${local.name}-public-rt" } +} + +resource "aws_route_table_association" "public" { + count = var.az_count + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +# --- fck-nat: a community-maintained NAT instance AMI. ~$3/mo on t4g.nano +# vs ~$35/mo for a managed NAT Gateway. See https://fck-nat.dev. --- + +data "aws_ami" "fck_nat" { + most_recent = true + owners = ["568608671756"] # fck-nat publisher + + # AWS provider v6 hard-fails most_recent lookups unless an owner is pinned + # via owner-id or image-id. owners=[] satisfies that, but the explicit + # filter is cheap belt-and-suspenders and keeps things working if a future + # version tightens the rule further. + filter { + name = "owner-id" + values = ["568608671756"] + } + + filter { + name = "name" + # AL2023 builds. The "fck-nat-nat64-*" prefix is a different (NAT64) + # variant — the dash anchors keep us on the plain NAT image. + values = ["fck-nat-al2023-*-arm64-ebs"] + } +} + +resource "aws_security_group" "fck_nat" { + name = "${local.name}-fck-nat" + description = "Allow private subnets to NAT through this instance" + vpc_id = aws_vpc.main.id + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = [var.vpc_cidr] + description = "From within VPC" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name}-fck-nat" } +} + +resource "aws_network_interface" "fck_nat" { + subnet_id = aws_subnet.public[0].id + security_groups = [aws_security_group.fck_nat.id] + source_dest_check = false + + tags = { Name = "${local.name}-fck-nat-eni" } +} + +resource "aws_eip" "fck_nat" { + domain = "vpc" + network_interface = aws_network_interface.fck_nat.id + tags = { Name = "${local.name}-fck-nat-eip" } + + depends_on = [aws_internet_gateway.main] +} + +resource "aws_instance" "fck_nat" { + ami = data.aws_ami.fck_nat.id + instance_type = var.fck_nat_instance_type + + network_interface { + network_interface_id = aws_network_interface.fck_nat.id + device_index = 0 + } + + metadata_options { + http_tokens = "required" + } + + tags = { Name = "${local.name}-fck-nat" } + + # Ensure the EIP is attached to the ENI before the instance starts using it + # — otherwise EIP association races with the instance's ENI attachment. + depends_on = [aws_eip.fck_nat] +} + +resource "aws_route_table" "private" { + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + network_interface_id = aws_network_interface.fck_nat.id + } + + tags = { Name = "${local.name}-private-rt" } +} + +resource "aws_route_table_association" "private" { + count = var.az_count + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private.id +} + +# --- Security group attached to the Lambda ENIs --- + +resource "aws_security_group" "lambda" { + name = "${local.name}-lambda" + description = "Egress for Lambda ENIs" + vpc_id = aws_vpc.main.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name}-lambda" } +} diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf new file mode 100644 index 0000000..cea2799 --- /dev/null +++ b/infra/terraform/outputs.tf @@ -0,0 +1,39 @@ +output "api_url" { + description = "Public HTTPS URL of the API" + value = "https://${local.api_fqdn}" +} + +output "region" { + description = "AWS region this stack is deployed in" + value = var.region +} + +output "ecr_repository_url" { + description = "Push the Lambda container image here" + value = aws_ecr_repository.lambda.repository_url +} + +output "api_lambda_name" { + value = aws_lambda_function.api.function_name +} + +output "worker_lambda_name" { + value = aws_lambda_function.worker.function_name +} + +output "sqs_queue_url" { + value = aws_sqs_queue.packages.url +} + +output "sqs_dlq_url" { + value = aws_sqs_queue.dlq.url +} + +output "rds_endpoint" { + value = aws_db_instance.main.address +} + +output "fck_nat_eip" { + description = "Public IP that all outbound Lambda traffic comes from" + value = aws_eip.fck_nat.public_ip +} diff --git a/infra/terraform/providers.tf b/infra/terraform/providers.tf new file mode 100644 index 0000000..7563505 --- /dev/null +++ b/infra/terraform/providers.tf @@ -0,0 +1,12 @@ +provider "aws" { + region = var.region + allowed_account_ids = var.allowed_account_ids + + default_tags { + tags = { + Project = "dumpus-api" + Environment = var.environment + ManagedBy = "terraform" + } + } +} diff --git a/infra/terraform/rds.tf b/infra/terraform/rds.tf new file mode 100644 index 0000000..5373006 --- /dev/null +++ b/infra/terraform/rds.tf @@ -0,0 +1,66 @@ +resource "aws_security_group" "rds" { + name = "${local.name}-rds" + description = "Postgres access from Lambdas" + vpc_id = aws_vpc.main.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name}-rds" } +} + +resource "aws_security_group_rule" "rds_from_lambda" { + type = "ingress" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_group_id = aws_security_group.rds.id + source_security_group_id = aws_security_group.lambda.id + description = "Postgres from Lambda ENIs" +} + +resource "aws_db_subnet_group" "main" { + name = "${local.name}-db" + subnet_ids = aws_subnet.private[*].id +} + +resource "aws_db_instance" "main" { + identifier = "${local.name}-postgres" + engine = "postgres" + engine_version = var.postgres_version + instance_class = var.postgres_instance_class + + allocated_storage = var.postgres_allocated_storage + max_allocated_storage = var.postgres_max_allocated_storage > 0 ? var.postgres_max_allocated_storage : null + storage_type = "gp3" + storage_encrypted = true + + db_name = var.postgres_db_name + username = var.postgres_username + password = random_password.postgres.result + + db_subnet_group_name = aws_db_subnet_group.main.name + vpc_security_group_ids = [aws_security_group.rds.id] + publicly_accessible = false + + multi_az = var.postgres_multi_az + backup_retention_period = 7 + backup_window = "02:00-03:00" + maintenance_window = "Mon:03:30-Mon:04:30" + + auto_minor_version_upgrade = true + deletion_protection = true + skip_final_snapshot = false + final_snapshot_identifier = "${local.name}-postgres-final-${formatdate("YYYYMMDDhhmm", timestamp())}" + + performance_insights_enabled = true + performance_insights_retention_period = 7 + + lifecycle { + ignore_changes = [final_snapshot_identifier, password] + } +} diff --git a/infra/terraform/secrets.tf b/infra/terraform/secrets.tf new file mode 100644 index 0000000..a402413 --- /dev/null +++ b/infra/terraform/secrets.tf @@ -0,0 +1,67 @@ +resource "random_password" "postgres" { + # Alphanumeric only: any of `:`, `?`, `&`, `=`, `@`, `/` would otherwise need + # URL-encoding before being spliced into the SQLAlchemy URL, and SQLAlchemy + # treats them as URL syntax if it sees them raw. 32 alnum chars give ~190 bits + # of entropy, which is plenty. + length = 32 + special = false +} + +resource "aws_secretsmanager_secret" "postgres_password" { + name = "${local.name}/postgres/password" + recovery_window_in_days = 7 +} + +resource "aws_secretsmanager_secret_version" "postgres_password" { + secret_id = aws_secretsmanager_secret.postgres_password.id + secret_string = random_password.postgres.result +} + +resource "aws_secretsmanager_secret" "postgres_url" { + name = "${local.name}/postgres/url" + description = "SQLAlchemy URL consumed by the app" + recovery_window_in_days = 7 +} + +resource "aws_secretsmanager_secret_version" "postgres_url" { + secret_id = aws_secretsmanager_secret.postgres_url.id + secret_string = format( + "postgresql+psycopg2://%s:%s@%s:%d/%s", + var.postgres_username, + random_password.postgres.result, + aws_db_instance.main.address, + aws_db_instance.main.port, + var.postgres_db_name, + ) +} + +resource "aws_secretsmanager_secret" "diswho_jwt_secret" { + name = "${local.name}/app/diswho-jwt-secret" + recovery_window_in_days = 7 +} + +resource "aws_secretsmanager_secret_version" "diswho_jwt_secret" { + secret_id = aws_secretsmanager_secret.diswho_jwt_secret.id + secret_string = var.diswho_jwt_secret + + # TF seeds the value on first apply. After that, rotate via + # `aws secretsmanager put-secret-value`; this prevents tofu apply from + # reverting the rotation. + lifecycle { + ignore_changes = [secret_string] + } +} + +resource "aws_secretsmanager_secret" "wh_url" { + name = "${local.name}/app/wh-url" + recovery_window_in_days = 7 +} + +resource "aws_secretsmanager_secret_version" "wh_url" { + secret_id = aws_secretsmanager_secret.wh_url.id + secret_string = var.wh_url + + lifecycle { + ignore_changes = [secret_string] + } +} diff --git a/infra/terraform/sqs.tf b/infra/terraform/sqs.tf new file mode 100644 index 0000000..f845d0a --- /dev/null +++ b/infra/terraform/sqs.tf @@ -0,0 +1,19 @@ +resource "aws_sqs_queue" "dlq" { + name = "${local.name}-packages-dlq" + message_retention_seconds = 14 * 24 * 60 * 60 # 14 days +} + +resource "aws_sqs_queue" "packages" { + name = "${local.name}-packages" + + # Has to comfortably exceed worker_lambda_timeout, plus a buffer for retries. + visibility_timeout_seconds = var.worker_lambda_timeout + 60 + + message_retention_seconds = 4 * 24 * 60 * 60 # 4 days + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.dlq.arn + # 1 retry then DLQ. Bumping it just keeps a poison message hot for longer. + maxReceiveCount = 2 + }) +} diff --git a/infra/terraform/terraform.ci.tfvars b/infra/terraform/terraform.ci.tfvars new file mode 100644 index 0000000..82fc0c8 --- /dev/null +++ b/infra/terraform/terraform.ci.tfvars @@ -0,0 +1,18 @@ +# Non-sensitive tfvars committed to the repo, used by the infra GitHub Actions +# workflow. Sensitive values (diswho_jwt_secret, wh_url) come in via TF_VAR_* +# env from GitHub secrets — never commit those here. + +region = "eu-west-1" +environment = "prod" + +allowed_account_ids = ["436630934006"] + +domain_name = "dumpus.app" +api_subdomain = "api" + +github_repository = "dumpus-app/dumpus-api" + +image_tag = "bootstrap" + +diswho_base_url = "https://diswho.androz2091.fr" +dl_zip_whitelisted_domains = "" diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example new file mode 100644 index 0000000..468a8eb --- /dev/null +++ b/infra/terraform/terraform.tfvars.example @@ -0,0 +1,32 @@ +# Copy to terraform.tfvars and fill in. Do NOT commit terraform.tfvars. + +region = "eu-west-1" +environment = "prod" + +# Strongly recommended: pin the account ID so a stray AWS_PROFILE pointing +# at the wrong account fails the plan instead of touching it. Get yours with +# `aws sts get-caller-identity --query Account --output text`. +# allowed_account_ids = ["123456789012"] + +# Must already exist as a Route53 hosted zone in this account. +domain_name = "dumpus.app" +api_subdomain = "api" + +# Secrets — better passed as env: TF_VAR_diswho_jwt_secret=... tofu apply +diswho_jwt_secret = "" +wh_url = "" +diswho_base_url = "" + +# Set to enable the GitHub Actions deploy role. +github_repository = "Androz2091/dumpus-api" + +# Initial image tag the Lambdas point at. After the first apply, CI's +# update-function-code calls take over and Terraform ignores image_uri +# changes, so this only matters for the very first deploy. +image_tag = "bootstrap" + +# Optional sizing overrides +# worker_lambda_memory = 3008 +# worker_ephemeral_storage_mb = 5120 +# worker_reserved_concurrency = 1 +# postgres_multi_az = false diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf new file mode 100644 index 0000000..142b2c0 --- /dev/null +++ b/infra/terraform/variables.tf @@ -0,0 +1,163 @@ +variable "region" { + description = "AWS region" + type = string + default = "eu-west-1" +} + +variable "allowed_account_ids" { + description = "If non-empty, refuse to plan/apply unless the resolved AWS credentials belong to one of these account IDs. Strongly recommended on shared workstations." + type = list(string) + default = [] +} + +variable "environment" { + description = "Environment name (used for tagging and resource naming)" + type = string + default = "prod" +} + +variable "name_prefix" { + description = "Prefix applied to all resource names" + type = string + default = "dumpus" +} + +# ---- DNS / TLS ---- + +variable "domain_name" { + description = "Apex domain (must already have a Route53 hosted zone)" + type = string + default = "dumpus.app" +} + +variable "api_subdomain" { + description = "Hostname for the public API" + type = string + default = "api" +} + +# ---- Networking ---- + +variable "vpc_cidr" { + type = string + default = "10.40.0.0/16" +} + +variable "az_count" { + description = "Number of AZs to spread subnets across" + type = number + default = 2 +} + +variable "fck_nat_instance_type" { + description = "Instance type for the fck-nat NAT replacement (~$3/mo on t4g.nano)" + type = string + default = "t4g.nano" +} + +# ---- Database ---- + +variable "postgres_version" { + type = string + default = "17.2" +} + +variable "postgres_instance_class" { + type = string + default = "db.t4g.micro" +} + +variable "postgres_allocated_storage" { + type = number + default = 20 +} + +variable "postgres_max_allocated_storage" { + description = "Storage autoscaling cap in GB (0 to disable)" + type = number + default = 100 +} + +variable "postgres_db_name" { + type = string + default = "dumpus" +} + +variable "postgres_username" { + type = string + default = "dumpus" +} + +variable "postgres_multi_az" { + type = bool + default = false +} + +# ---- Lambda sizing ---- + +variable "image_tag" { + description = "Container image tag deployed to both Lambdas. CI overrides per-deploy via UpdateFunctionCode." + type = string + default = "bootstrap" +} + +variable "api_lambda_memory" { + description = "MB. 512 is plenty for Flask + apig-wsgi." + type = number + default = 512 +} + +variable "api_lambda_timeout" { + description = "Seconds. API Gateway HTTP API caps at 30." + type = number + default = 30 +} + +variable "worker_lambda_memory" { + description = "MB. Pandas + zip processing are memory-hungry; 3008 is the sweet spot." + type = number + default = 3008 +} + +variable "worker_lambda_timeout" { + description = "Seconds. Lambda hard cap is 900." + type = number + default = 900 +} + +variable "worker_ephemeral_storage_mb" { + description = "MB of /tmp space for the worker (Discord zips). 512..10240." + type = number + default = 5120 +} + +variable "worker_reserved_concurrency" { + description = "Cap on concurrent worker Lambdas. 0 = no reservation (account-level pool). New AWS accounts have a 10-execution ceiling and reject any reservation until you request a Lambda concurrency quota increase. Set > 0 once that's lifted." + type = number + default = 0 +} + +# ---- App secrets / config ---- + +variable "wh_url" { + description = "Optional Discord webhook URL the app pings for internal notifications (new package, errored package, etc.). Leave blank to disable." + type = string + sensitive = true + default = "" +} + +variable "diswho_jwt_secret" { + type = string + sensitive = true + default = "" +} + +variable "diswho_base_url" { + type = string + default = "" +} + +variable "dl_zip_whitelisted_domains" { + type = string + default = "" +} diff --git a/infra/terraform/versions.tf b/infra/terraform/versions.tf new file mode 100644 index 0000000..8026253 --- /dev/null +++ b/infra/terraform/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = "~> 3.2" + } + } +} diff --git a/requirements.txt b/requirements.txt index 33540ba..2426985 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,12 @@ -amqp==5.1.1 +apig-wsgi==2.20.0 async-timeout==4.0.2 autopep8==2.0.2 -billiard==3.6.4.0 blinker==1.6.2 -celery==5.2.7 +boto3==1.35.0 certifi==2024.7.4 cffi==1.15.1 charset-normalizer==3.1.0 click==8.1.3 -click-didyoumean==0.3.0 -click-plugins==1.1.1 -click-repl==0.2.0 cryptocode==0.1 cryptography==41.0.1 Deprecated==1.2.14 @@ -22,18 +18,15 @@ importlib-resources==5.12.0 itsdangerous==2.1.2 Jinja2==3.1.6 joblib==1.2.0 -kombu==5.2.4 limits==3.5.0 markdown-it-py==3.0.0 MarkupSafe==2.1.2 mdurl==0.1.2 -nltk==3.9.1 numpy==1.24.3 ordered-set==4.1.0 orjson==3.9.15 packaging==23.1 pandas==2.0.1 -prompt-toolkit==3.0.38 psycopg2-binary==2.9.6 pycodestyle==2.10.0 pycparser==2.21 @@ -43,7 +36,6 @@ PyJWT==2.7.0 python-dateutil==2.8.2 python-dotenv==1.0.0 pytz==2023.3 -redis==4.5.5 regex==2023.6.3 requests==2.32.2 rich==13.4.2 @@ -54,7 +46,6 @@ tqdm==4.66.3 typing_extensions==4.6.2 tzdata==2023.3 urllib3==2.2.2 -vine==5.0.0 waitress==2.1.2 wcwidth==0.2.6 Werkzeug==2.3.4 diff --git a/scripts/download-ntk.py b/scripts/download-ntk.py deleted file mode 100644 index 247ce06..0000000 --- a/scripts/download-ntk.py +++ /dev/null @@ -1,3 +0,0 @@ -# download_nltk.py -import nltk -nltk.download('vader_lexicon') diff --git a/src/app.py b/src/app.py index 4dcc6d4..63a8d31 100644 --- a/src/app.py +++ b/src/app.py @@ -5,9 +5,9 @@ from flask_limiter import Limiter -# make sure tasks is imported before db -# as env is loaded from tasks (so the celery worker can use it) -from tasks import handle_package +# Import tasks before db so dotenv loads before SQLAlchemy reads POSTGRES_URL. +import tasks # noqa: F401 +from enqueue import enqueue_package from db import PackageProcessStatus, SavedPackageData, Session, fetch_package_status, fetch_package_data, fetch_package_rank from sqlite import generate_demo_database @@ -148,8 +148,7 @@ def process_link(): session.close() - handle_package.apply_async( - args=[id, package_id, link], queue='regular_process') + enqueue_package(id, package_id, link) return jsonify(res), 200 diff --git a/src/celeryconfig.py b/src/celeryconfig.py deleted file mode 100644 index 9276e40..0000000 --- a/src/celeryconfig.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -broker_url=os.getenv('RABBITMQ_URL') -task_ignore_result=True -task_acks_late=True diff --git a/src/enqueue.py b/src/enqueue.py new file mode 100644 index 0000000..5eb8036 --- /dev/null +++ b/src/enqueue.py @@ -0,0 +1,32 @@ +"""Pluggable queue dispatch. + +Production (Lambda) uses SQS. Local dev defaults to `sync` — process the package +inline on the request thread, which is good enough at our volume and means +no broker is needed to run the API locally. +""" +import json +import os + + +def enqueue_package(package_status_id, package_id, link): + backend = os.getenv('QUEUE_BACKEND', 'sync') + + if backend == 'sqs': + import boto3 + sqs = boto3.client('sqs') + sqs.send_message( + QueueUrl=os.environ['SQS_QUEUE_URL'], + MessageBody=json.dumps({ + 'package_status_id': package_status_id, + 'package_id': package_id, + 'link': link, + }), + ) + return + + if backend == 'sync': + from tasks import process_package + process_package(package_status_id, package_id, link) + return + + raise ValueError(f'Unknown QUEUE_BACKEND: {backend}') diff --git a/src/lambda_handlers/__init__.py b/src/lambda_handlers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/lambda_handlers/api.py b/src/lambda_handlers/api.py new file mode 100644 index 0000000..5f72906 --- /dev/null +++ b/src/lambda_handlers/api.py @@ -0,0 +1,13 @@ +"""API Gateway HTTP API → Flask via apig-wsgi.""" +import sys +from pathlib import Path + +# `src/` must be on sys.path so the existing flat-module imports (`from tasks import ...`) +# keep working when this module is loaded from `src/lambda_handlers/api.py`. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import secrets_loader # noqa: F401, E402 — runs at import to fetch SM secrets into os.environ +from apig_wsgi import make_lambda_handler # noqa: E402 +from app import app # noqa: E402 + +handler = make_lambda_handler(app) diff --git a/src/lambda_handlers/worker.py b/src/lambda_handlers/worker.py new file mode 100644 index 0000000..b849f5f --- /dev/null +++ b/src/lambda_handlers/worker.py @@ -0,0 +1,38 @@ +"""SQS-triggered worker Lambda. + +Receives a batch of SQS records, each with a JSON body containing +{package_status_id, package_id, link}. Calls process_package for each. + +Failures are reported via the partial-batch-response protocol so successful +records aren't redelivered. The Lambda event source mapping must be +configured with FunctionResponseTypes=["ReportBatchItemFailures"]. +""" +import json +import sys +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import secrets_loader # noqa: F401, E402 — runs at import to fetch SM secrets into os.environ +from tasks import process_package # noqa: E402 + + +def handler(event, context): + failures = [] + + for record in event.get('Records', []): + try: + body = json.loads(record['body']) + process_package( + body['package_status_id'], + body['package_id'], + body['link'], + worker_name='regular_process', + ) + except Exception as e: + print(f"record {record.get('messageId')} failed: {e}") + traceback.print_exc() + failures.append({'itemIdentifier': record['messageId']}) + + return {'batchItemFailures': failures} diff --git a/src/secrets_loader.py b/src/secrets_loader.py new file mode 100644 index 0000000..41df000 --- /dev/null +++ b/src/secrets_loader.py @@ -0,0 +1,41 @@ +"""Hydrate os.environ from AWS Secrets Manager at module import time. + +Lambda runtime gets a single env var `SECRETS_ARN_MAP`, which is a JSON +object like: + + {"POSTGRES_URL": "arn:aws:secretsmanager:...", "DISCORD_SECRET": "arn:..."} + +For each entry, this module fetches the secret's value and sets the env var +of the same name — but only if it isn't already set, so you can still +override locally. + +No-op when SECRETS_ARN_MAP is unset (local dev). +""" +import json +import os + + +def _load() -> None: + raw = os.getenv("SECRETS_ARN_MAP") + if not raw: + return + + arn_map = json.loads(raw) + if not arn_map: + return + + # Import boto3 only when we actually need it — avoids paying the import + # cost in local dev where this module is a no-op. + import boto3 + + client = boto3.client("secretsmanager") + for env_name, arn in arn_map.items(): + if env_name in os.environ: + continue # don't override an explicitly-set value + if not arn: + continue + resp = client.get_secret_value(SecretId=arn) + os.environ[env_name] = resp["SecretString"] + + +_load() diff --git a/src/tasks.py b/src/tasks.py index 9210283..9ea877d 100644 --- a/src/tasks.py +++ b/src/tasks.py @@ -1,8 +1,6 @@ from dotenv import load_dotenv load_dotenv() -from celery import Celery, current_task - import traceback import pandas as pd @@ -48,18 +46,15 @@ from wh import send_internal_notification -from nltk.sentiment import SentimentIntensityAnalyzer -sia = SentimentIntensityAnalyzer() - +# TODO: restore real sentiment scoring. Stubbed to a constant so we can +# drop NLTK + the vader_lexicon download from the Lambda image while we +# decide whether sentiment is worth the ~50MB of data and the cold-start +# cost. Re-enable by reinstating SentimentIntensityAnalyzer here and the +# nltk install step in Dockerfile.lambda. def count_sentiments(contents): - sentiments = [] - for content in contents: - if not content: - continue - score = sia.polarity_scores(str(content))["compound"] - if score != 0: - sentiments.append(score) - return sum(sentiments) / len(sentiments) if len(sentiments) > 0 else 0 + for _ in contents: + pass + return 0.5 def find_user_root(zip_namelist): """ @@ -113,8 +108,6 @@ def find_analytics_file(zip_namelist): # Look for any JSON file in a folder that could be analytics return next((name for name in zip_namelist if re.search(r'/analytics.*\.json$', name)), None) -app = Celery(config_source='celeryconfig') - def download_file(package_status_id, package_id, link, session): # check if file exists in tmp path = get_package_zip_path(package_id) @@ -905,25 +898,26 @@ def read_analytics_file(package_status_id, package_id, link, session): return analytics_line_count -@app.task() -def handle_package(package_status_id, package_id, link): +def process_package(package_status_id, package_id, link, worker_name='regular_process'): + """Process a package end-to-end. + + Invoked by enqueue_package(QUEUE_BACKEND=sync) for local dev and by the SQS + worker Lambda in production. worker_name lets premium-only work skip the + regular-process path; we only run regular_process today. + """ print(f'handling package {package_id} with link {link}') session = Session() package_status = session.query(PackageProcessStatus).filter(PackageProcessStatus.id == package_status_id).first() if not package_status: print('package not found') return - + if package_status.is_cancelled: print('package is cancelled, skipping') return - # regular_process or premium_process - worker_name = current_task.request.hostname - if package_status.is_upgraded and worker_name.startswith('regular_process'): print('package is upgraded and worker is regular, skipping') - # the package has already been added to the premium worker queue return try: @@ -956,3 +950,5 @@ def handle_package(package_status_id, package_id, link): finally: remove_file(package_id) session.close() + +