Add CI check that the previous release works against the new schema

.github/workflows/build.yml

@@ -419,13 +419,61 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
+      # Skip on tag (release) builds: this commit was already tested on its branch/main
+      # push, so re-running here is redundant. See backwards-compatibility-test for why
+      # release builds must not run these gated tests.
       - name: Test migrations
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
         run: ./docker/ci/test_migrations.sh
 
+  backwards-compatibility-test:
+    name: Backwards compatibility test
+    runs-on: ubuntu-24.04
+    needs: [build]
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Download digests
+        uses: actions/download-artifact@v8
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-amd64
+
+      # This sets the environment variable referenced in the docker-compose file
+      # for the image to use for the webapp container to the digest of the image
+      # that was built in the build job.
+      - name: Set webapp image
+        working-directory: ${{ runner.temp }}/digests/webapp
+        run: |
+          IMAGE="${{needs.build.outputs.webapp-repo}}$(printf '@sha256:%s' *)"
+          echo "$IMAGE"
+          echo "WEBAPP_IMAGE=$IMAGE" >> $GITHUB_ENV
+
+      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
+      - name: Disable network offload
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      # Skip on tag (release) builds. On a release the just-published tag becomes the "latest"
+      # release, so the script would resolve the previous-release image to this build's own
+      # X.Y.Z tag — which the push job hasn't created yet — deadlocking the release.
+      - name: Test backwards compatibility
+        if: ${{ !startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./docker/ci/test_backwards_compatibility.sh
+
   push:
     name: Tag & Push Images
     runs-on: ubuntu-24.04
-    needs: [build, integration-test, unit-test, migration-test]
+    needs: [build, integration-test, unit-test, migration-test, backwards-compatibility-test]
     permissions:
       contents: read
       packages: write

README.md

@@ -781,10 +781,10 @@ Setting `PALACE_TEST_DATABASE_EXTERNAL_SCHEMA` to `true` tells the tests that th
 been applied to the provided database (by the current code), so the fixtures use that schema as-is,
 without dropping and recreating it from the current models. With database creation left enabled (the
 default), each worker clones the provided database as a template into its own per-worker database, so
-the suite can still run in parallel. If you additionally set `PALACE_TEST_DATABASE_CREATE_DATABASE` to
-`false`, the provided database is used directly, which requires the tests to be run serially. This is
-used by the backwards-compatibility CI check, which runs a previous release's tests against a schema
-built by the current code.
+the suite can still run in parallel. This is how the backwards-compatibility CI check runs a previous
+release's tests against a schema built by the current code. If you additionally set
+`PALACE_TEST_DATABASE_CREATE_DATABASE` to `false`, the provided database is used directly, which
+requires the tests to be run serially.
 
 ### Override `pytest` Arguments
 

docker/ci/test_backwards_compatibility.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# This script checks that the PREVIOUS released version of the application still works
+# against the database schema produced by the CURRENT code. It exercises our online
+# migration requirement: a release's database must keep working with version N-1 of the
+# code, because during a deploy the new migrations run while the previous version's
+# webservers are still serving traffic.
+#
+# It works by:
+#  (1) Finding the previous release (the latest GitHub release) and its published
+#      circ-webapp image.
+#  (2) Initializing a fresh database with the CURRENT image, which builds the new schema.
+#  (3) Running the previous release's database tests (pytest -m db) against that schema in
+#      external-schema mode (PALACE_TEST_DATABASE_EXTERNAL_SCHEMA), so the older code
+#      exercises the new schema. If those tests fail, the migration is not backwards
+#      compatible.
+#
+# The current image is provided via the WEBAPP_IMAGE environment variable (as in the other
+# docker CI jobs). The previous release image can be overridden via PREV_RELEASE_IMAGE
+# (useful for running this script locally).
+
+set -uo pipefail
+
+COMPOSE_FILES=(-f docker-compose.yml -f docker/ci/test_backwards_compatibility.yml)
+
+# Run a docker compose command for this check.
+compose_cmd() {
+  docker compose "${COMPOSE_FILES[@]}" --progress quiet "$@"
+}
+
+# Run a command in a container with the palace virtualenv activated.
+run_in_container() {
+  local container="$1"
+  shift
+  compose_cmd run --rm --no-deps "${container}" /bin/bash -c "source env/bin/activate && $*"
+}
+
+cleanup() {
+  compose_cmd down --remove-orphans >/dev/null 2>&1
+}
+
+fail() {
+  echo "::error::$1"
+  exit "${2:-1}"
+}
+
+# (1) Resolve the previous release image.
+if [[ -z "${PREV_RELEASE_IMAGE:-}" ]]; then
+  # In CI gh infers the repo from $GITHUB_REPOSITORY; locally it infers it from the git
+  # remote of the current directory.
+  gh_release_args=(release view --json tagName --jq '.tagName')
+  if [[ -n "${GITHUB_REPOSITORY:-}" ]]; then
+    gh_release_args+=(--repo "${GITHUB_REPOSITORY}")
+  fi
+
+  # Resolve the previous release's tag. We deliberately do NOT fail the job when this lookup
+  # fails: `gh release view` exits non-zero both when there is genuinely no prior release (a
+  # legitimate skip) and on transient errors (auth hiccup, rate limit, network blip). But we
+  # don't fail *open silently*, we let gh's error flow to the job log and check its exit status, and on
+  # any failure we emit a GitHub Actions ::warning:: before skipping. That makes the gate no-op
+  # visible at the PR/checks level instead of quietly passing.
+  prev_tag="$(gh "${gh_release_args[@]}")"
+  gh_status=$?
+  if [[ ${gh_status} -ne 0 || -z "${prev_tag}" ]]; then
+    echo "::warning::Backwards-compatibility check skipped: could not resolve the previous release (gh exit ${gh_status}). The gate did NOT run for this build -- see gh's error above. A transient gh/auth/network failure or a genuine absence of releases both land here."
+    exit 0
+  fi
+
+  PREV_RELEASE_IMAGE="ghcr.io/thepalaceproject/circ-webapp:${prev_tag#v}"
+fi
+export PREV_RELEASE_IMAGE
+echo "Previous release image: ${PREV_RELEASE_IMAGE}"
+
+if [[ -z "${WEBAPP_IMAGE:-}" ]]; then
+  fail "WEBAPP_IMAGE is not set; it must point at the current build's webapp image."
+fi
+
+trap cleanup EXIT
+
+# (2) Build the current schema by initializing a fresh database with the current image.
+compose_cmd up -d pg os minio redis || fail "Could not start service containers."
+run_in_container webapp "./bin/util/initialize_instance" \
+  || fail "Failed to initialize the database with the current image."
+
+# (3) Run the previous release's database tests against the new schema. In external-schema
+# mode each xdist worker clones the freshly-built schema into its own database, so the suite
+# runs in parallel (-n auto).
+compose_cmd pull --quiet webapp-prev || fail "Could not pull ${PREV_RELEASE_IMAGE}."
+echo "Running the previous release's database tests against the current schema ..."
+if ! run_in_container webapp-prev \
+  "uv sync --frozen --active && pytest --no-cov -n auto -m db --ignore=tests/migration tests"; then
+  fail "Previous release tests failed against the current schema: the migration is not backwards compatible."
+fi
+
+echo "The previous release works against the current schema 🎉"

docker/ci/test_backwards_compatibility.yml

@@ -0,0 +1,20 @@
+# Compose overlay for the backwards-compatibility check; see test_backwards_compatibility.sh.
+#
+# The `webapp` service (from docker-compose.yml, the image built by the current commit) is
+# used to build the new schema. `webapp-prev` runs a previously released image and re-runs
+# its database tests against that schema in external-schema mode.
+services:
+  webapp-prev:
+    # Inherit the full test environment (database/search/storage/redis URLs, etc.) from the
+    # webapp service so the previous release's test suite can run, then point it at the
+    # previous release's published image and turn on external-schema mode.
+    extends:
+      file: docker-compose.yml
+      service: webapp
+    image: "${PREV_RELEASE_IMAGE}"
+    environment:
+      PALACE_TEST_DATABASE_EXTERNAL_SCHEMA: "true"
+      # External-schema mode with database creation enabled clones the externally-built
+      # schema into a per-worker database, so the previous release's suite can run in
+      # parallel (-n auto); see DatabaseCreationFixture.
+      PALACE_TEST_DATABASE_CREATE_DATABASE: "true"