diff --git a/.gitignore b/.gitignore index c07db91..8df0e70 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,13 @@ # cookbook entry "How do I override env config locally?". *.local +# 1.x → 3.x migration intermediates, all secret-bearing: env.3x is the +# converter export from `task upgrade_prep` on the 1.x stack; the others are +# produced by `task env:migrate`. +env.3x +.env.symfony.migrated +.env.symfony.infra-advisory + # Operator-generated by `task host:resources` — host-specific resource limits. compose.resource-limits.yml .playwright-mcp/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 804ed3d..bfcec34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,10 @@ See the [v2.x.x — Skipped](#v2xx---skipped) entry below for why this major ski templates. - `task env:diff` compares your `.env.symfony` against the example shipped in the currently-pinned API image. -- `task env:migrate` rewrites a 1.x `.env.docker.local` into the v3 layout. +- `task env:migrate` stages a v3 `.env.symfony` from `env.3x` (the export the 1.x + `task upgrade_prep` produces via the 2.8 API's `app:utils:convert-env-to-3x`) when present — + splitting off the infrastructure advisory — and falls back to the sed rewrite of a 1.x + `.env.docker.local` otherwise. - Per-service `.env..local` override layer for site-specific tuning; gitignored. - Compose profiles (`COMPOSE_PROFILES=mariadb,traefik`) gate built-in services — drop `mariadb` for an external DB, drop `traefik` for an external proxy. @@ -38,6 +41,16 @@ See the [v2.x.x — Skipped](#v2xx---skipped) entry below for why this major ski — return only when the stack is genuinely ready (no more `sleep 20` race). - New `task db:backup` (online `mariadb-dump` to `./backup/.sql.gz`) and `task db:upgrade` (idempotent `mariadb-upgrade`). +- The mariadb service sets `MARIADB_AUTO_UPGRADE=1`, so `mariadb-upgrade` runs automatically on + the first 11.4 boot against a carried-over 10.x data dir (the 1.x → 3.x case) — no manual + step required for the system-table upgrade. It also refreshes the healthcheck user, rescuing + data dirs old enough to predate it. +- The mariadb service sets `stop_grace_period: 1m`, so the engine gets room for a clean shutdown + instead of being SIGKILLed mid-flush by the 10s default — important right before a major bump. +- `UPGRADE.md` restructured around the 2.8 converter: a pre-upgrade checklist that exports the + running 1.x configuration (`task upgrade_prep`) *before* the stack is stopped, converter-first + env migration with the sed rename demoted to a fallback, MariaDB auto-upgrade and data-volume + continuity notes, and the `rm docker-compose.yml` step needed before `git checkout`. - New MariaDB diagnostics: `task db:metrics`, `db:processes`, `db:errors`. - New `task php:opcache` — operator report on the FPM pool's OPcache (memory, interned strings, cached-key headroom, hit rate, restarts, preload) with warnings mapped to the diff --git a/README.md b/README.md index 2b973d1..bd030b9 100644 --- a/README.md +++ b/README.md @@ -296,8 +296,10 @@ reasoning. #### How do I upgrade the bundled MariaDB across a major version? See [UPGRADE.md](UPGRADE.md). The 1.x → 3.x section has the recipe (it covers the 10.x → 11.4 -jump that came with the 3.0 release); the same `task db:backup` → `task db:upgrade` → -update `DATABASE_URL` `serverVersion=` flow applies to any future major bump. +jump that came with the 3.0 release). The mariadb service sets `MARIADB_AUTO_UPGRADE=1`, so +`mariadb-upgrade` runs automatically the first time a newer image starts against the existing +data dir; the manual flow for any future major bump is `task db:backup` → bump the image tag → +`task db:upgrade` (explicit, idempotent re-run) → update `DATABASE_URL` `serverVersion=`. #### How do I switch from Let's Encrypt to a custom certificate? @@ -1248,7 +1250,7 @@ Lifecycle Bootstrap and env-file tooling env:init Bootstrap .env.symfony from the API image env:diff Compare .env.symfony against the image's shipped example - env:migrate Convert a 1.x .env.docker.local to .env.symfony.migrated + env:migrate Stage .env.symfony.migrated from env.3x (preferred) or a 1.x .env.docker.local env:traefik Interactive .env.traefik setup (alias: traefik_env) Operations diff --git a/Taskfile.yml b/Taskfile.yml index 51be63b..f23b5ff 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -284,18 +284,66 @@ tasks: rm -f "$tmp" env:migrate: - desc: Convert a 1.x .env.docker.local into a 3.x-shaped .env.symfony.migrated + desc: Stage a 3.x-shaped .env.symfony.migrated — from env.3x (preferred) or a 1.x .env.docker.local cmds: - | + OUT=.env.symfony.migrated + + # Preferred path: env.3x produced by `task upgrade_prep` on the 1.x + # stack (the API's app:utils:convert-env-to-3x command). It already + # carries the loaded values under their 3.x names AND the admin/client + # config.json conversion the sed path below can't see — so when it's + # present we just split off the trailing infrastructure-advisory + # comment block (COMPOSE_*/PHP_*/NGINX_*/MARIADB_* notes) into a + # reminder, and the rest becomes .env.symfony directly. + if [ -f env.3x ]; then + echo "Found env.3x from the 1.x upgrade prep — using it as the source." + # env.3x ends with a trailing advisory block the converter + # introduces with a fixed marker line; everything above it is the + # application env. Split at the marker: app env to .env.symfony, + # the advisory to its own file (surfaced below, not buried in the + # app env). + ADVISORY=.env.symfony.infra-advisory + rm -f "$ADVISORY" + awk -v adv="$ADVISORY" ' + /^# The following loaded variables are NOT read/ { infra = 1 } + infra { print > adv; next } + { print } + ' env.3x > "$OUT" + echo "Wrote $OUT (from env.3x)." + if [ -s "$ADVISORY" ]; then + echo "" + echo "Infrastructure variables were split into $ADVISORY." + echo "They do NOT belong in .env.symfony — distribute them across the" + echo "per-service files of the 3.x layout:" + echo " COMPOSE_* -> .env PHP_* -> .env.php" + echo " NGINX_* -> .env.nginx MARIADB_* -> .env.mariadb" + fi + echo "" + echo "Review with: diff -u env.3x $OUT" + echo "Apply with: mv $OUT .env.symfony" + echo "" + echo "Still set by hand: DATABASE_URL serverVersion=11.4.10-MariaDB" + echo "(after the MariaDB upgrade — see UPGRADE.md step 6)." + exit 0 + fi + + # Fallback path: no env.3x (older 1.x without the converter command, or + # the stack is already down). Convert a 1.x .env.docker.local by sed — + # an APP_ strip plus the per-site renames the converter would do. if [ -f .env.docker.local ]; then SRC=.env.docker.local elif [ -f .env.local ]; then SRC=.env.local else - echo "Error: neither .env.docker.local nor .env.local exists." + echo "Error: no env.3x, .env.docker.local or .env.local found." + echo "On the 1.x stack, prefer 'task upgrade_prep' to produce env.3x." exit 1 fi - OUT=.env.symfony.migrated + echo "No env.3x found — falling back to the sed conversion of $SRC." + echo "(The env.3x path from 'task upgrade_prep' on 1.x is preferred:" + echo " it also converts the admin/client config.json, which sed can't.)" + echo "" # Strip APP_ prefix from every key EXCEPT the framework-defined trio # (APP_ENV, APP_SECRET, APP_DEBUG) — Symfony recognises those by name, # and renaming them silently breaks env-driven debug/secret/env config. diff --git a/UPGRADE.md b/UPGRADE.md index 8094bcd..a6dc6f4 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -22,6 +22,18 @@ staging host before touching production. The on-disk data files survive the upgr (MariaDB 11 reads 10.x InnoDB tablespaces; the JWT keys and uploaded media don't move), but the schema gets rewritten by Doctrine migrations and the operator env layout is restructured. +### Table of contents + +- [What's changed at a glance](#whats-changed-at-a-glance) +- [Pre-upgrade checklist (while still on 1.x)](#pre-upgrade-checklist-while-still-on-1x) +- [Step 1 — Stop the 1.x stack](#step-1--stop-the-1x-stack) +- [Step 2 — Update the repo](#step-2--update-the-repo) +- [Step 3 — Rewrite the env configuration](#step-3--rewrite-the-env-configuration) +- [Step 4 — Upgrade MariaDB 10.x to 11.4](#step-4--upgrade-mariadb-10x-to-114) +- [Step 5 — First boot](#step-5--first-boot) +- [Post-upgrade validation](#post-upgrade-validation) +- [Rollback](#rollback) + ### What's changed at a glance The full breaking-change list is in [CHANGELOG.md](CHANGELOG.md) under @@ -33,67 +45,81 @@ The full breaking-change list is in [CHANGELOG.md](CHANGELOG.md) under services + their `OS2DISPLAY_VERSION_ADMIN` / `OS2DISPLAY_VERSION_CLIENT` pins are gone. - **Symfony env vars lose the `APP_` prefix** (except `APP_ENV` and `APP_SECRET`, which Symfony defines). `APP_DATABASE_URL` becomes `DATABASE_URL`, `APP_JWT_PASSPHRASE` becomes - `JWT_PASSPHRASE`, etc. Full 56-key rename list in upstream `display-api-service`'s `UPGRADE.md` - § 2.1; `task env:migrate` performs the rename automatically. + `JWT_PASSPHRASE`, etc. The 2.8 API ships `app:utils:convert-env-to-3x`, which does the rename + for you (and converts the admin/client `config.json`) — this repo's `task upgrade_prep` runs + it; see the pre-upgrade checklist below. - **One env file per service.** The single `.env.docker.local` is gone, replaced by `.env.symfony` + `.env.php` + `.env.nginx` + `.env.mariadb` + `.env.traefik` + the compose-orchestration `.env`. See [README § Configuration files](README.md#configuration-files). -- **MariaDB 10.x → 11.4 LTS.** Major version bump. Binary-compatible at the data file level - (MariaDB 11 reads 10.x InnoDB tablespaces) but requires `mariadb-upgrade` and a `serverVersion` - update in `DATABASE_URL`. +- **MariaDB 10.x → 11.4 LTS.** Major version bump, binary-compatible at the data file level + (MariaDB 11 reads 10.x InnoDB tablespaces). The 3.x mariadb service sets `MARIADB_AUTO_UPGRADE=1`, + so `mariadb-upgrade` runs automatically on the first 11.4 boot against the carried-over data; + you still update `serverVersion` in `DATABASE_URL`. - **Compose profiles replace `INTERNAL_DATABASE` / `INTERNAL_PROXY`.** Use `COMPOSE_PROFILES=mariadb,traefik` (default), or drop a token to run with external infra. - **Image registry switched** from `itkdev/os2display-*` (Docker Hub) to `ghcr.io/os2display/display-api-service{,-nginx}` (GHCR). -### Step-by-step - -#### 1. Pre-flight +### Pre-upgrade checklist (while still on 1.x) -On the production host, before touching anything: +Everything in this section runs on the **1.x** checkout, with the stack **still up**. The +configuration export (`task upgrade_prep`) reads the running application and fetches the live +admin/client `config.json` — it cannot run once the stack is stopped. Confirm prerequisites +first: ```bash -# Confirm prerequisites are in place. docker --version # 20.10+ docker compose version # v2 (the integrated subcommand, not legacy docker-compose) task --version # v3+ id # confirm UID 1042 / GID 1042 if you've followed the README - -# Confirm the host has disk headroom for backups. -df -h . +df -h . # headroom for the database dump ``` Read the [Caveats and foot-guns](README.md#caveats-and-foot-guns) section in the README before proceeding. The cert-file SAN-coverage caveat and the `DATABASE_URL` `serverVersion=` caveat in particular bite operators on this migration. -#### 2. Take backups - -```bash -# Database — full dump, gzipped, online (no service downtime). -task db:backup -ls -lh backup/ -# 20260505T141522Z.sql.gz - -# Env files + JWT keys — copy aside in case a rollback is needed. -mkdir -p /tmp/os2display-1x-backup -cp .env .env.docker.local .env.local /tmp/os2display-1x-backup/ # whatever subset you have -cp -r jwt/ /tmp/os2display-1x-backup/ -``` +Then, on the 1.x stack: + +- [ ] **Be on the final 1.x release running 2.8 API images.** The export command ships with + `os2display-api-service` 2.8.0. Set `COMPOSE_VERSION_API=2.8.0` (or a later 2.x) in + `.env.docker.local` and `task install` to pull and recreate. The last 1.x release of this repo + (v1.2.0) provides `task upgrade_check` / `task upgrade_prep`; upgrade to it first if you are on + an earlier 1.x. +- [ ] **Run the pre-flight check:** `task upgrade_check`. It confirms the api image provides + `app:utils:convert-env-to-3x` and prints the bundled MariaDB volume name + (`_mariadb`). **Record that volume name** — the 3.x stack reuses it in + place, and only does so when the 3.x `.env` keeps the same `COMPOSE_PROJECT_NAME`. A different + project name silently boots an *empty* database (see step 4). +- [ ] **Back up the database:** `task backup_db` (1.x), and keep the dump somewhere off-host. +- [ ] **Export the configuration in 3.x shape:** `task upgrade_prep`. It writes `env.3x` — the + loaded env converted to 3.x names *plus* the admin/client `config.json` conversion. The + trailing advisory block lists infrastructure variables (`COMPOSE_*`, `PHP_*`, `NGINX_*`, + `MARIADB_*`) that move to per-service files in 3.x, never into the application env. +- [ ] **Copy env files + JWT keys aside** in case a rollback is needed: + + ```bash + mkdir -p /tmp/os2display-1x-backup + cp .env .env.docker.local .env.local env.3x /tmp/os2display-1x-backup/ # whatever subset you have + cp -r jwt/ /tmp/os2display-1x-backup/ + ``` + +`env.3x` contains every application secret (`APP_SECRET`, database and OIDC credentials, ...). It +is gitignored on both branches; treat it like a credentials file and it will survive the branch +switch in step 2 untouched. You do **not** need to copy `./media/` aside for the rollback. It's a host bind-mount, not a docker named volume — `task purge`, `task down --volumes`, and `docker compose down --volumes` all leave it on disk. The `./media` directory survives the upgrade-then-rollback round-trip in -place; the v1 database dump's filename references still match the files on disk after a -restore. +place; the v1 database dump's filename references still match the files on disk after a restore. That said, the media bind-mount is also where uploaded user data lives. If you have a snapshot tool (LVM, ZFS, btrfs, cloud volume snapshots), take a host-level snapshot here for paranoia — protects against unrelated disk failures during the maintenance window, not against the -upgrade itself. Same for `./jwt/`: bind-mounted, preserved across compose lifecycle, but a -copy aside in `/tmp/os2display-1x-backup/` is cheap. +upgrade itself. Same for `./jwt/`: bind-mounted, preserved across compose lifecycle, but a copy +aside in `/tmp/os2display-1x-backup/` is cheap. -#### 3. Stop the 1.x stack +### Step 1 — Stop the 1.x stack ```bash task stop # bring containers down; preserves volumes @@ -101,17 +127,38 @@ task stop # bring containers down; preserves volumes Don't run `task purge` or `task down --volumes` — those delete the database. -#### 4. Update the repo +This is the **last shutdown of the 10.x MariaDB**; the next process to read its data files is +11.4. A server SIGKILLed mid-flush leaves a dirty data dir, and crash recovery across a major +version is unsupported. The 1.x compose has no extended stop grace, so confirm the shutdown was +clean before continuing: + +```bash +docker compose -f docker-compose.yml logs mariadb | tail -n 20 # expect "mariadbd: Shutdown complete" +``` + +If the log shows the server was killed before completing shutdown (large or busy database), bring +it back up under 1.x (`task up`), let it settle, and stop it again giving the container more time +(`docker compose -f docker-compose.yml stop -t 120 mariadb`). + +### Step 2 — Update the repo ```bash git fetch +rm -f docker-compose.yml # see note git checkout release/3.0.0 ``` -#### 5. Rewrite env config +The 1.x stack generated an untracked `docker-compose.yml` (via `task _dc_compile`). The 3.x +branch *tracks* a file at that path, so `git checkout` aborts with "untracked working tree files +would be overwritten" unless you remove the generated one first. It is regenerated from the +tracked compose file in 3.x — nothing of value is lost. (1.x v1.2.0 gitignores this file, but an +already-present untracked copy still blocks the checkout, so the `rm` is needed regardless of +which 1.x you came from.) + +### Step 3 — Rewrite the env configuration The 1.x layout was a single `.env.docker.local` consumed by compose substitution. v3 uses -per-service `env_file:` directives. Walk the new layout once: +per-service `env_file:` directives: | File | Purpose | |---|---| @@ -122,60 +169,90 @@ per-service `env_file:` directives. Walk the new layout once: | `.env.mariadb` | MariaDB credentials. | | `.env.traefik` | Traefik dashboard auth, Let's Encrypt email, cert provider. | -Bootstrap each: +Build `.env.symfony` from the `env.3x` you exported in the pre-upgrade checklist, then bootstrap +the rest: ```bash -# Compose orchestration: copy from the example, then port over your old domain, image -# versions, and (the renamed) profile selection. -cp .env.example .env -$EDITOR .env -# OS2DISPLAY_SERVER_DOMAIN=... (was COMPOSE_SERVER_DOMAIN in 1.x) -# OS2DISPLAY_VERSION_API= (was COMPOSE_VERSION_API; see -# .env.example for the current pin) -# COMPOSE_PROFILES=mariadb,traefik (replaces INTERNAL_DATABASE=true and -# INTERNAL_PROXY=true; combine into a profile -# list) - -# Symfony app config: convert your old .env.docker.local. task env:migrate strips the -# APP_ prefix from every key except the framework-defined trio (APP_ENV / APP_SECRET / -# APP_DEBUG, all of which Symfony recognises by name), writing the result to -# .env.symfony.migrated for review. +# 1. Application config from env.3x. task env:migrate detects env.3x, splits off the +# infrastructure advisory (into .env.symfony.infra-advisory) and writes the rest to +# .env.symfony.migrated for review. task env:migrate -diff -u .env.docker.local .env.symfony.migrated # sanity check -$EDITOR .env.symfony.migrated -# - Strip any 1.x compose-orchestration block at the top of the file -# (COMPOSE_PROJECT_NAME, COMPOSE_VERSION_API, COMPOSE_SERVER_DOMAIN, -# INTERNAL_DATABASE, INTERNAL_PROXY). Those don't belong in .env.symfony — -# their v3 equivalents live in .env (created above). -# - Set DATABASE_URL serverVersion to "11.4.10-MariaDB" (post-MariaDB upgrade — see step 6). -# - Rename per-site customisation keys to their v3 prefixes (the script can't -# infer this — only the APP_ → bare-name strip is automatic): -# TOUCH_BUTTON_REGIONS → ADMIN_TOUCH_BUTTON_REGIONS -# REJSEPLANEN_API_KEY → ADMIN_REJSEPLANEN_APIKEY -# SHOW_SCREEN_STATUS → ADMIN_SHOW_SCREEN_STATUS -# DATA_PULL_INTERVAL → CLIENT_PULL_STRATEGY_INTERVAL -# SCHEDULING_INTERVAL → CLIENT_SCHEDULING_INTERVAL -# - The image's bundled .env supplies sane ADMIN_* / CLIENT_* defaults; the renames -# above are only needed for keys you'd customised in 1.x. +diff -u env.3x .env.symfony.migrated # sanity check mv .env.symfony.migrated .env.symfony -# Run `task env:diff` after editing to spot any keys upstream added that you haven't set. - -# Per-service runtime config + Traefik: task env:init creates the per-service env files -# (.env.php, .env.nginx, .env.mariadb, .env.traefik) from .env..example. It detects -# .env.symfony already exists from env:migrate and skips it (no FORCE=1 needed). +# 2. Per-service files + .env. task env:init creates .env (prompts for the domain) and the +# per-service .env.{php,nginx,mariadb,traefik} from their .example templates. It sees the +# .env.symfony you just wrote and leaves it untouched (no FORCE=1 needed). task env:init +# 3. Finish .env: port over your old image version and profile selection. +$EDITOR .env +# OS2DISPLAY_VERSION_API= (was COMPOSE_VERSION_API; see .env.example) +# COMPOSE_PROFILES=mariadb,traefik (replaces INTERNAL_DATABASE / INTERNAL_PROXY) +# COMPOSE_PROJECT_NAME= (MUST match — see step 4; from task upgrade_check) + +# 4. Finish .env.symfony. +$EDITOR .env.symfony +# - Set DATABASE_URL serverVersion to "11.4.10-MariaDB" (post-MariaDB upgrade — see step 4). +# env.3x carries the old 10.x serverVersion through verbatim; this is the one value env:migrate +# leaves for you. +# - Distribute the keys from .env.symfony.infra-advisory: COMPOSE_* -> .env, PHP_* -> .env.php, +# NGINX_* -> .env.nginx, MARIADB_* -> .env.mariadb. + +# 5. Match the bundled MariaDB credentials to the existing data dir. $EDITOR .env.mariadb -# Match credentials to the user/password/database in your old .env.docker.local — -# they MUST equal the user / password / db in DATABASE_URL above. Mismatched credentials -# means Doctrine can't connect, AND mariadb won't re-initialise its data dir with new ones. +# MARIADB_USER / MARIADB_PASSWORD / MARIADB_DATABASE / MARIADB_ROOT_PASSWORD MUST equal the +# values baked into the carried-over data dir (i.e. what 1.x used, and what DATABASE_URL +# encodes). On a non-empty data dir the mariadb entrypoint IGNORES these — it does not +# re-initialise — so a mismatch does not error at container start; it surfaces later as +# Doctrine/tooling auth failures. Set them to the 1.x values. -# Traefik: re-run the interactive setup task, or edit .env.traefik by hand. +# 6. Traefik: re-run the interactive setup, or edit .env.traefik by hand. task env:traefik + +# 7. Spot any keys the pinned image added that you haven't set. +task env:diff ``` -##### Media upload size: align all three layers +
+Manual fallback (no `env.3x` — pre-2.8 images, or stack already stopped) + +If you never produced `env.3x` (the 1.x install predates the converter, or the stack is already +down), `task env:migrate` falls back to a sed conversion of `.env.docker.local`: it strips the +`APP_` prefix from every key except the framework-defined trio (`APP_ENV` / `APP_SECRET` / +`APP_DEBUG`) and writes `.env.symfony.migrated`. This path **cannot** see the admin/client +`config.json`, so you also convert those by hand and apply the per-site renames the converter +would have done: + +```bash +task env:migrate # sed path when env.3x is absent +diff -u .env.docker.local .env.symfony.migrated +$EDITOR .env.symfony.migrated +``` + +- Strip any 1.x compose-orchestration block (`COMPOSE_*`, `INTERNAL_*`) from the top — those + belong in `.env` (`COMPOSE_PROJECT_NAME` / `COMPOSE_PROFILES`), not `.env.symfony`. +- Rename per-site customisation keys to their v3 prefixes: + + ```text + TOUCH_BUTTON_REGIONS → ADMIN_TOUCH_BUTTON_REGIONS + REJSEPLANEN_API_KEY → ADMIN_REJSEPLANEN_APIKEY + SHOW_SCREEN_STATUS → ADMIN_SHOW_SCREEN_STATUS + DATA_PULL_INTERVAL → CLIENT_PULL_STRATEGY_INTERVAL + SCHEDULING_INTERVAL → CLIENT_SCHEDULING_INTERVAL + ``` + +- Convert your old admin/client `config.json` with the 3.x command (after first boot, or in a + one-off container): `bin/console app:utils:convert-config-json-to-env --type=admin ` + and `--type=client `. + +The image's bundled `.env` supplies sane `ADMIN_*` / `CLIENT_*` defaults; the renames above are +only needed for keys you'd customised in 1.x. Then `mv .env.symfony.migrated .env.symfony` and +rejoin the main path at step 2 (`task env:init`). + +
+ +#### Media upload size: align all three layers v3 adds an app-level cap on media uploads, `MEDIA_MAX_UPLOAD_SIZE_MB` (default `200`, in MiB), enforced by the Symfony validator on `Media::$file`. Because uploads cross @@ -189,43 +266,50 @@ the inequality intact: NGINX_MAX_BODY_SIZE >= PHP_POST_MAX_SIZE >= PHP_UPLOAD_MAX_FILESIZE >= MEDIA_MAX_UPLOAD_SIZE_MB ``` -`task env:migrate` carries `MEDIA_MAX_UPLOAD_SIZE_MB` over from the image's bundled -`.env`, so it lands in `.env.symfony` automatically. Verify with `task env:diff` after -`env:init` — if the diff shows the key only on the image side, your migrated file -predates this var and you should copy the line over by hand. +`MEDIA_MAX_UPLOAD_SIZE_MB` comes from the image's bundled `.env`, which `task env:init` extracts +into `.env.symfony` (the converter export carries it too). Verify with `task env:diff` after +`env:init` — if the diff shows the key only on the image side, copy the line over by hand. -#### 6. MariaDB 10.x → 11.4 upgrade +### Step 4 — Upgrade MariaDB 10.x to 11.4 -The bundled MariaDB is `mariadb:11.4.10` in 3.x (was `mariadb:10.x` in 1.x). Three things must -happen for a clean cut: +The bundled MariaDB is `mariadb:11.4.10` in 3.x (was `mariadb:10.x` in 1.x). The 3.x mariadb +service sets `MARIADB_AUTO_UPGRADE=1`, so the system-table upgrade runs automatically; you just +bring the new image up against the carried-over data and verify. ```bash -# 1. Backup is done from step 2. Verify it's recent and readable: -ls -lh backup/ -gunzip -t backup/.sql.gz # checks gzip integrity - -# 2. Pull the new mariadb image, restart the DB container alone, run mariadb-upgrade. +# 1. Confirm the backup from the pre-upgrade checklist is recent and readable. +ls -lh backup/ db_backups/ 2>/dev/null # 1.x writes to db_backups/ +gunzip -t backup/.sql.gz 2>/dev/null || true + +# 2. Confirm the data volume carried over and the project name matches. +# A mismatched COMPOSE_PROJECT_NAME resolves a DIFFERENT (empty) volume — the +# stack would come up green with no data. The name below must be the one +# task upgrade_check recorded on 1.x. +docker volume ls | grep mariadb + +# 3. Pull the new image and start the DB container alone. With MARIADB_AUTO_UPGRADE=1 the +# entrypoint version-compares the data dir and runs mariadb-upgrade before accepting +# connections; on a large database this can take a while, so watch the log. docker compose pull mariadb docker compose up -d mariadb -docker compose logs -f mariadb # wait for "ready for connections" - -task db:upgrade # idempotent; also auto-runs by the - # mariadb image entrypoint on first - # start with new data, but explicit - # is better here. +docker compose logs -f mariadb # wait for the upgrade to finish and + # "ready for connections" -# 3. The DATABASE_URL serverVersion edit in .env.symfony from step 5 above must match -# 11.4.10-MariaDB now. Doctrine reads it to pick its SQL dialect; mismatch → wrong -# queries (most often: incorrect JSON or function syntax). +# 4. Optional explicit re-run / verification. Idempotent — a no-op if the auto-upgrade already +# ran. +task db:upgrade ``` -If `task db:upgrade` reports incompatible objects, **stop here**. Restore from the dump (step -2), revert the mariadb image tag in `docker-compose.yml`, and debug on a non-prod box. The -common cause is custom stored procedures or views with reserved-word identifiers in 11.4 that -were valid in 10.x — Doctrine + Symfony won't typically have any, but custom feed types and -admin commands sometimes do. +Then confirm the `DATABASE_URL` `serverVersion` edit from step 3 reads `11.4.10-MariaDB`. +Doctrine uses it to pick its SQL dialect; a mismatch produces wrong queries (most often +incorrect JSON or function syntax). -#### 7. First boot +If the upgrade reports incompatible objects, **stop here.** Restore from the dump, revert the +mariadb image tag, and debug on a non-prod box. The common cause is custom stored procedures or +views with reserved-word identifiers in 11.4 that were valid in 10.x — Doctrine + Symfony won't +typically have any, but custom feed types and admin commands sometimes do. + +### Step 5 — First boot ```bash task install # pulls the rest of the images, runs `bin/console app:update` @@ -244,7 +328,7 @@ nginx/screen-client tier comes up, so the stack never serves traffic against an schema. The interactive tenant + user prompts can be skipped (Ctrl-D) if your existing tenants / admin user are already in the data. -#### 8. Validation +### Post-upgrade validation Walk through these checks in order. Any failure → roll back from the backup before debugging further. @@ -273,21 +357,27 @@ task purge # WIPES the bundled mariadb data volume # wipes the redis cache (harmless; it rebuilds). # Does NOT touch ./media or ./jwt — those are # bind mounts, preserved through compose lifecycle. +rm -f docker-compose.yml # 3.x tracks it; remove before checking out 1.x git checkout # the tag or branch you came from -docker compose up -d mariadb # bring up the OLD mariadb alone -docker compose logs -f mariadb # wait for "ready for connections" - -# Restore the dump. -gunzip < /path/to/backup/.sql.gz \ - | docker compose exec -T mariadb mariadb -u root -p"$(grep ^MARIADB_ROOT_PASSWORD= .env.docker.local | cut -d= -f2-)" -# Restore the rest of your old config (jwt/ and media/ are still on disk — -# nothing to copy back). +# Restore the 1.x env files first — `task up` regenerates the 1.x compose file +# (task _dc_compile) from them, and the restore below reads credentials from +# .env.docker.local. jwt/ and media/ are still on disk; nothing to copy back. cp /tmp/os2display-1x-backup/.env.docker.local . cp /tmp/os2display-1x-backup/.env.local . # if it existed -# Bring the rest of the 1.x stack up. -task install # under the 1.x release that's now checked out +task up # 1.x: regenerates docker-compose.yml and starts the + # stack with a fresh (empty) mariadb volume +docker compose -f docker-compose.yml logs -f mariadb # wait for "ready for connections" + +# Load the dump taken in the pre-upgrade checklist. `task backup_db` (1.x) writes a plain, +# single-database SQL file to db_backups/ — restore it into that same database. +docker compose -f docker-compose.yml exec -T mariadb \ + mariadb -u root -p"$(grep ^MARIADB_ROOT_PASSWORD= .env.docker.local | cut -d= -f2-)" \ + "$(grep ^MARIADB_DATABASE= .env.docker.local | cut -d= -f2-)" \ + < /path/to/db_backups/db_backup_.sql + +task cc # clear the 1.x app cache against the restored data ``` What survives the rollback in place: @@ -301,16 +391,18 @@ What survives the rollback in place: What gets destroyed and restored: -- **MariaDB** — named volume, removed by `task purge --volumes`, restored by piping the dump - back in. **This is why `task db:backup` at step 2 is non-negotiable** — without it, the - rollback path can't reach v1's data. +- **MariaDB** — named volume, removed by `task purge`, restored by loading the dump back in. + **This is why the database backup in the pre-upgrade checklist is non-negotiable** — without + it, the rollback path can't reach v1's data. - **Redis cache** — named volume, removed and rebuilt empty. v1 application repopulates on first request. Don't try to roll back without restoring the dump — `app:update`'s 3.x migrations may have applied schema changes that 1.x's `app:update` doesn't reverse, so a "tag revert + bring the 3.x-mutated DB up under 1.x" path will fail silently (wrong queries, missing columns) or -loudly (Doctrine refusing to start). +loudly (Doctrine refusing to start). Note too that once 11.4 has upgraded the data dir in place +(step 4), the 10.x server can no longer read it — the rollback restores the dump into a fresh +volume rather than reusing the upgraded files. ## Future migrations diff --git a/docker-compose.yml b/docker-compose.yml index e4ec0fd..c8af408 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -133,12 +133,27 @@ services: required: true - path: .env.mariadb.local required: false + environment: + # Run mariadb-upgrade automatically when the server version recorded in + # the data dir is older than the running image (e.g. the 1.x 10.x → 3.x + # 11.4 jump). Idempotent: the entrypoint version-compares first and does + # nothing on an already-current data dir, so it's safe to leave on for + # future major bumps too. It also refreshes the healthcheck user and its + # /var/lib/mysql/.my-healthcheck.cnf, which rescues data dirs old enough + # to predate that user. `task db:upgrade` remains available for an + # explicit, operator-driven run. + - MARIADB_AUTO_UPGRADE=1 healthcheck: test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] interval: 30s timeout: 5s retries: 5 start_period: 30s + # The default 10s stop grace can SIGKILL a server still flushing a dirty + # InnoDB buffer pool. A hard kill leaves the data dir needing crash + # recovery — unsupported across a major-version bump — so give the engine + # room for a clean shutdown, especially right before the 10.x → 11.4 cut. + stop_grace_period: 1m volumes: - mariadb:/var/lib/mysql:rw