feat(api): allow choosing MySQL or PostgreSQL via DB_ENGINE (#151)

vvatelot · cursoragent · web-flow · commit ba6ac89f4410 · 2026-07-15T17:37:54.000+02:00
* feat(api): allow choosing MySQL or PostgreSQL via DB_ENGINE Developers can now pick sqlite, mysql, or postgres at startup through a single DB_ENGINE setting, with Docker Compose profiles and local dev containers aligned on the same configuration. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(api): support emoji and punycode domains in analysis tasks (#150) * fix(api): support emoji and punycode domains in analysis tasks WebPage validation stores URLs as Unicode, which breaks requests for emoji domains like xn--3s8h30f.ws. Use AnyHttpUrl for punycode conversion and pass the encoded URL to the worker queue. Fixes cnumr/EcoIndex#416 Co-authored-by: Cursor <cursoragent@cursor.com> * fix(api): include request error details when URL pre-check fails Expose SSL, timeout, and DNS errors in the unreachable URL response instead of empty parentheses. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com> * fix(api): keep DATABASE_URL typed as str for ty compatibility Use an empty default and resolve it in the model validator so type checkers accept Settings().DATABASE_URL where a str is required. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(api): run alembic migrations from the api project directory Alembic needs projects/ecoindex_api as the working directory so it can find alembic.ini and load the local .env during init-dev-project. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(api): start dev DB before running alembic migrations Ensure mysql/postgres containers are up before migration-upgrade and document which DB_HOST to use for local dev versus Docker Compose. Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
diff --git a/components/ecoindex/config/settings.py b/components/ecoindex/config/settings.py
@@ -1,5 +1,35 @@
+from typing import Literal
+from urllib.parse import quote_plus
+
+from pydantic import model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+DbEngine = Literal["sqlite", "mysql", "postgres"]
+
+
+def build_database_url(
+    *,
+    engine: DbEngine = "sqlite",
+    host: str = "localhost",
+    port: int | None = None,
+    user: str = "ecoindex",
+    password: str = "ecoindex",
+    name: str = "ecoindex",
+) -> str:
+    if engine == "sqlite":
+        return "sqlite+aiosqlite:///db.sqlite3"
+
+    credentials = f"{quote_plus(user)}:{quote_plus(password)}"
+
+    if engine == "mysql":
+        db_port = port or 3306
+        return (
+            f"mysql+aiomysql://{credentials}@{host}:{db_port}/{name}?charset=utf8mb4"
+        )
+
+    db_port = port or 5432
+    return f"postgresql+asyncpg://{credentials}@{host}:{db_port}/{name}"
+
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env")
@@ -12,7 +42,13 @@ class Settings(BaseSettings):
     CORS_ALLOWED_METHODS: list = ["*"]
     CORS_ALLOWED_ORIGINS: list = ["*"]
     DAILY_LIMIT_PER_HOST: int = 0
-    DATABASE_URL: str = "sqlite+aiosqlite:///db.sqlite3"
+    DATABASE_URL: str = ""
+    DB_ENGINE: DbEngine = "sqlite"
+    DB_HOST: str = "localhost"
+    DB_PORT: int | None = None
+    DB_USER: str = "ecoindex"
+    DB_PASSWORD: str = "ecoindex"
+    DB_NAME: str = "ecoindex"
     DEBUG: bool = False
     DOCKER_CONTAINER: bool = False
     ENABLE_SCREENSHOT: bool = False
@@ -40,3 +76,16 @@ class Settings(BaseSettings):
     TZ: str = "Europe/Paris"
     WAIT_AFTER_SCROLL: int = 3
     WAIT_BEFORE_SCROLL: int = 3
+
+    @model_validator(mode="after")
+    def resolve_database_url(self) -> "Settings":
+        if not self.DATABASE_URL:
+            self.DATABASE_URL = build_database_url(
+                engine=self.DB_ENGINE,
+                host=self.DB_HOST,
+                port=self.DB_PORT,
+                user=self.DB_USER,
+                password=self.DB_PASSWORD,
+                name=self.DB_NAME,
+            )
+        return self
diff --git a/projects/ecoindex_api/.env.template b/projects/ecoindex_api/.env.template
@@ -1,11 +1,20 @@
 # API_PORT=8001
 # API_VERSION=latest
 # DAILY_LIMIT_PER_HOST=10
-# DB_HOST=db
+# DB_ENGINE=sqlite
+# DB_ENGINE=mysql
+# DB_ENGINE=postgres
+# Local dev (task api:start-dev): DB_HOST=localhost
+# Docker Compose: DB_HOST=db-mysql or DB_HOST=db-postgres
+# DB_HOST=localhost
+# DB_HOST=db-mysql
+# DB_HOST=db-postgres
 # DB_NAME=ecoindex
 # DB_PASSWORD=ecoindex
 # DB_PORT=3306
+# DB_PORT=5432
 # DB_USER=ecoindex
+# DATABASE_URL=
 # DEBUG=1
 # ENABLE_SCREENSHOT=1
 # EXCLUDED_HOSTS='["localhost","127.0.0.1"]'
diff --git a/projects/ecoindex_api/README.md b/projects/ecoindex_api/README.md
@@ -30,7 +30,7 @@ The API specification can be found in the [documentation](projects/ecoindex_api/
 
 With this docker setup you get 5 services running that are enough to make it all work:
 
-- `db`: A MySQL instance
+- `db-mysql` or `db-postgres`: database instance (selected with `DB_ENGINE`)
 - `api`: The API instance running FastAPI application
 - `worker`: The RQ task worker that runs ecoindex analysis
 - `valkey`: The [Valkey](https://valkey.io/) instance (Redis-compatible) used by the RQ worker and API cache
@@ -40,7 +40,16 @@ With this docker setup you get 5 services running that are enough to make it all
 
 ```bash
 cp docker-compose.yml.template docker-compose.yml && \
-docker  compose up -d
+cp .env.template .env && \
+task api:docker-up-mysql -- -d
+```
+
+For PostgreSQL instead:
+
+```bash
+cp docker-compose.yml.template docker-compose.yml && \
+cp .env.template .env && \
+task api:docker-up-postgres -- -d
 ```
 
 Every services should start normaly, then you can go to:
@@ -62,7 +71,13 @@ Here are the environment variables you can configure in your `.env` file:
 | API                 | `CORS_ALLOWED_ORIGINS`     | `*`                                | See [MDN web doc](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin)                                                                                                                                                                                                                                                                                                                   |
 | API                 | `EXCLUDED_HOSTS`           | `["localhost", "127.0.0.1"]`       | You can configure a list of hosts that will be excluded from the analysis.                                                                                                                                                                                                                                                                                                                                                 |
 | API, Worker         | `DAILY_LIMIT_PER_HOST`     | 0                                  | When this variable is set, it won't be possible for a same host to make more request than defined in the same day to avoid overload. If the variable is set, you will get a header `x-remaining-daily-requests: 6` in your response. It is used for the POST methods. If you reach your authorized request quota for the day, the next requests will give you a 429 response. If the variable is set to 0, no limit is set |
-| API, Worker         | `DATABASE_URL`             | `sqlite+aiosqlite:///./sql_app.db` | If you run your mysql instance on a dedicated server, you can configure it with your credentials. By default, it uses an sqlite database when running in local                                                                                                                                                                                                                                                             |  |
+| API, Worker         | `DB_ENGINE`                | `sqlite`                           | Database backend: `sqlite`, `mysql` or `postgres`. Used to build `DATABASE_URL` when it is not set explicitly.                                                                                                                                                                                                                                              |
+| API, Worker         | `DB_HOST`                  | `localhost`                        | Database host. Use `db-mysql` or `db-postgres` in Docker Compose.                                                                                                                                                                                                                                                                                          |
+| API, Worker         | `DB_PORT`                  | `3306` / `5432`                    | Database port. Defaults to the standard port of the selected engine when omitted.                                                                                                                                                                                                                                                                           |
+| API, Worker         | `DB_USER`                  | `ecoindex`                         | Database user.                                                                                                                                                                                                                                                                                                                                              |
+| API, Worker         | `DB_PASSWORD`              | `ecoindex`                         | Database password.                                                                                                                                                                                                                                                                                                                                          |
+| API, Worker         | `DB_NAME`                  | `ecoindex`                         | Database name.                                                                                                                                                                                                                                                                                                                                              |
+| API, Worker         | `DATABASE_URL`             | built from `DB_ENGINE`             | Optional explicit SQLAlchemy URL. When set, it overrides `DB_ENGINE` and related variables. Examples: `sqlite+aiosqlite:///db.sqlite3`, `mysql+aiomysql://user:pass@host/db?charset=utf8mb4`, `postgresql+asyncpg://user:pass@host:5432/db`                                                                                                                |
 | API, Worker         | `SENTRY_DSN`               | ``                                 | If you want to use [Sentry](https://sentry.io/) to monitor your application, set this variable with your project DSN.                                                                                                                                                                                                                                                                                                      |
 | API, Worker         | `SENTRY_ENVIRONMENT`       | ``                                 | Optional Sentry environment name (e.g. `production`, `staging`). If not set, defaults to `development` when `DEBUG=True`, otherwise `production`.                                                                                                                                                                                                                                                                    |
 | API, Worker         | `SENTRY_TRACES_SAMPLE_RATE`| `0.0`                              | Fraction of transactions to send to Sentry for performance monitoring (0.0 to 1.0). Set to `0.1` in production to sample 10% of requests.                                                                                                                                                                                                                                                                                 |
@@ -132,7 +147,13 @@ task api:init-dev-project # Initialize API dev environment (Playwright, .env, mi
 
 ### Run the API locally
 
-Valkey and RustFS are started automatically via Docker. Then run:
+Valkey and RustFS are started automatically via Docker. Set `DB_ENGINE` in `.env` to choose the database:
+
+- `sqlite` (default): no database container, file stored locally
+- `mysql`: starts a local MySQL container on port 3306
+- `postgres`: starts a local PostgreSQL container on port 5432
+
+Then run:
 
 ```bash
 task api:start-dev
diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml
@@ -121,18 +121,32 @@ tasks:
     silent: true
 
   docker-up:
-    desc: Start the docker-compose API
+    desc: Start the docker-compose API (uses DB_ENGINE from .env, default mysql)
     deps: [init-env, init-docker-compose]
     cmds:
-      - docker compose up {{.CLI_ARGS}}
+      - bash scripts/docker_compose_up.sh {{.CLI_ARGS}}
+    silent: true
+
+  docker-up-mysql:
+    desc: Start the docker-compose API with MySQL
+    deps: [init-env, init-docker-compose]
+    cmds:
+      - DB_ENGINE=mysql DB_HOST=db-mysql DB_PORT=3306 bash scripts/docker_compose_up.sh {{.CLI_ARGS}}
+    silent: true
+
+  docker-up-postgres:
+    desc: Start the docker-compose API with PostgreSQL
+    deps: [init-env, init-docker-compose]
+    cmds:
+      - DB_ENGINE=postgres DB_HOST=db-postgres DB_PORT=5432 bash scripts/docker_compose_up.sh {{.CLI_ARGS}}
     silent: true
 
   docker-down:
     desc: Stop the docker-compose API
     preconditions:
       - test -f docker-compose.yml
     cmds:
-      - docker compose down {{.CLI_ARGS}}
+      - docker compose --profile mysql --profile postgres down {{.CLI_ARGS}}
     silent: true
 
   docker-exec:
@@ -156,23 +170,19 @@ tasks:
     desc: Create a new alembic migration
     cmds:
       - uv run --package ecoindex_api alembic revision --autogenerate -m "{{.CLI_ARGS}}"
-    dir: ../..
     silent: true
 
   migration-upgrade:
     desc: Upgrade the database to the last migration
+    deps: [start-dev-infra]
     cmds:
       - uv run --package ecoindex_api alembic upgrade head
-    dir: ../..
     silent: true
 
   start-dev-infra:
     internal: true
     cmds:
       - bash scripts/start_dev_infra.sh
-    status:
-      - docker inspect ecoindex-dev-valkey --format '{{.State.Running}}' 2>/dev/null | grep -q true
-      - docker inspect ecoindex-dev-rustfs --format '{{.State.Running}}' 2>/dev/null | grep -q true
     silent: true
 
   start-worker:
@@ -226,7 +236,7 @@ tasks:
 
   start-dev:
     deps: [start-backend, start-worker, start-rq-dashboard]
-    desc: Start the backend, the worker and the RQ dashboard
+    desc: Start the backend, the worker and the RQ dashboard (set DB_ENGINE in .env)
     cmds:
       - echo "Starting the backend, worker and RQ dashboard (http://localhost:{{.RQ_DASHBOARD_PORT}})"
     silent: true
@@ -254,7 +264,7 @@ tasks:
         pkill -f "uvicorn ecoindex.backend.main:app" 2>/dev/null || true
 
         echo "Stopping Docker containers..."
-        docker rm -f ecoindex-dev-valkey ecoindex-dev-rustfs 2>/dev/null || true
+        docker rm -f ecoindex-dev-valkey ecoindex-dev-rustfs ecoindex-dev-mysql ecoindex-dev-postgres 2>/dev/null || true
         echo "Local development environment stopped."
     silent: true
 
diff --git a/projects/ecoindex_api/docker-compose.yml.template b/projects/ecoindex_api/docker-compose.yml.template
@@ -1,9 +1,10 @@
 services:
-  db:
-    image: mysql
+  db-mysql:
+    profiles: ["mysql"]
+    image: mysql:8
     restart: always
     volumes:
-      - db:/var/lib/mysql
+      - db-mysql:/var/lib/mysql
     environment:
       MYSQL_DATABASE: ${DB_NAME:-ecoindex}
       MYSQL_USER: ${DB_USER:-ecoindex}
@@ -17,6 +18,24 @@ services:
       retries: 10
       interval: 2s
 
+  db-postgres:
+    profiles: ["postgres"]
+    image: postgres:16-alpine
+    restart: always
+    volumes:
+      - db-postgres:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: ${DB_NAME:-ecoindex}
+      POSTGRES_USER: ${DB_USER:-ecoindex}
+      POSTGRES_PASSWORD: ${DB_PASSWORD:-ecoindex}
+    ports:
+      - "${DB_PORT:-5432}:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
+      timeout: 5s
+      retries: 10
+      interval: 2s
+
   backend:
     image: vvatelot/ecoindex-api-backend:${API_VERSION:-latest}
     restart: always
@@ -25,7 +44,12 @@ services:
     ports:
       - "${API_PORT:-8001}:8000"
     environment:
-      DATABASE_URL: mysql+aiomysql://${DB_USER:-ecoindex}:${DB_PASSWORD:-ecoindex}@${DB_HOST:-db}/${DB_NAME:-ecoindex}?charset=utf8mb4
+      DB_ENGINE: ${DB_ENGINE:-mysql}
+      DB_HOST: ${DB_HOST:-db-mysql}
+      DB_PORT: ${DB_PORT:-}
+      DB_USER: ${DB_USER:-ecoindex}
+      DB_PASSWORD: ${DB_PASSWORD:-ecoindex}
+      DB_NAME: ${DB_NAME:-ecoindex}
       DEBUG: ${DEBUG:-0}
       REDIS_CACHE_HOST: ${REDIS_CACHE_HOST:-valkey}
       SCREENSHOT_FILESYSTEM_PATH: ${SCREENSHOT_FILESYSTEM_PATH:-/code/screenshots}
@@ -39,8 +63,12 @@ services:
       SCREENSHOT_S3_SECRET_ACCESS_KEY: ${SCREENSHOT_S3_SECRET_ACCESS_KEY:-ecoindex-secret-key-change-me}
       TZ: ${TZ:-Europe/Paris}
     depends_on:
-      db:
+      db-mysql:
         condition: service_healthy
+        required: false
+      db-postgres:
+        condition: service_healthy
+        required: false
       rustfs-init:
         condition: service_completed_successfully
       valkey:
@@ -54,7 +82,12 @@ services:
     env_file:
       - .env
     environment:
-      DATABASE_URL: mysql+aiomysql://${DB_USER:-ecoindex}:${DB_PASSWORD:-ecoindex}@${DB_HOST:-db}/${DB_NAME:-ecoindex}?charset=utf8mb4
+      DB_ENGINE: ${DB_ENGINE:-mysql}
+      DB_HOST: ${DB_HOST:-db-mysql}
+      DB_PORT: ${DB_PORT:-}
+      DB_USER: ${DB_USER:-ecoindex}
+      DB_PASSWORD: ${DB_PASSWORD:-ecoindex}
+      DB_NAME: ${DB_NAME:-ecoindex}
       DEBUG: ${DEBUG:-0}
       REDIS_CACHE_HOST: ${REDIS_CACHE_HOST:-valkey}
       RQ_WORKERS: ${RQ_WORKERS:-3}
@@ -70,8 +103,12 @@ services:
       TZ: ${TZ:-Europe/Paris}
       ENABLE_SCREENSHOT: ${ENABLE_SCREENSHOT:-0}
     depends_on:
-      db:
+      db-mysql:
+        condition: service_healthy
+        required: false
+      db-postgres:
         condition: service_healthy
+        required: false
       rustfs-init:
         condition: service_completed_successfully
       valkey:
@@ -120,6 +157,7 @@ services:
     restart: "no"
 
 volumes:
-  db:
+  db-mysql:
+  db-postgres:
   valkey:
   rustfs_data:
diff --git a/projects/ecoindex_api/docker/backend/dockerfile b/projects/ecoindex_api/docker/backend/dockerfile
@@ -25,7 +25,7 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY projects/ecoindex_api/dist/$wheel $wheel
 RUN pip install --no-cache-dir $wheel
-RUN pip install --no-cache-dir aiomysql gunicorn
+RUN pip install --no-cache-dir aiomysql asyncpg gunicorn
 
 RUN rm -rf $wheel requirements.txt /tmp/dist /var/lib/{apt,dpkg,cache,log}/
 
diff --git a/projects/ecoindex_api/docker/worker/dockerfile b/projects/ecoindex_api/docker/worker/dockerfile
@@ -22,7 +22,7 @@ RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
 COPY projects/ecoindex_api/dist/$wheel $wheel
 RUN pip install --no-cache-dir $wheel
-RUN pip install --no-cache-dir aiomysql
+RUN pip install --no-cache-dir aiomysql asyncpg
 
 RUN playwright install chromium --with-deps
 
diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml
@@ -46,7 +46,9 @@ worker = [
     "playwright-stealth>=1.0.6",
 ]
 dev = [
+    "aiomysql>=0.2.0",
     "aiosqlite>=0.19.0",
+    "asyncpg>=0.29.0",
     "rq-dashboard",
     "typing-extensions>=4.8.0",
     "watchdog>=6.0.0",
diff --git a/projects/ecoindex_api/scripts/docker_compose_up.sh b/projects/ecoindex_api/scripts/docker_compose_up.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+API_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$API_DIR"
+
+if [ -f .env ]; then
+  set -a
+  # shellcheck disable=SC1091
+  . ./.env
+  set +a
+fi
+
+DB_ENGINE="${DB_ENGINE:-mysql}"
+
+case "$DB_ENGINE" in
+  mysql)
+    export DB_ENGINE
+    export DB_HOST="${DB_HOST:-db-mysql}"
+    export DB_PORT="${DB_PORT:-3306}"
+    docker compose --profile mysql up "$@"
+    ;;
+  postgres)
+    export DB_ENGINE
+    export DB_HOST="${DB_HOST:-db-postgres}"
+    export DB_PORT="${DB_PORT:-5432}"
+    docker compose --profile postgres up "$@"
+    ;;
+  sqlite)
+    echo "DB_ENGINE=sqlite is not supported in Docker Compose. Use mysql or postgres." >&2
+    exit 1
+    ;;
+  *)
+    echo "Unknown DB_ENGINE: $DB_ENGINE" >&2
+    exit 1
+    ;;
+esac
diff --git a/projects/ecoindex_api/scripts/start_dev_infra.sh b/projects/ecoindex_api/scripts/start_dev_infra.sh
diff --git a/projects/ecoindex_api/scripts/wait_db.sh b/projects/ecoindex_api/scripts/wait_db.sh
diff --git a/test/components/ecoindex/config/test_settings.py b/test/components/ecoindex/config/test_settings.py
diff --git a/uv.lock b/uv.lock
