diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da26d05ceb..b98d517a1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -419,13 +419,61 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # Skip on tag (release) builds: this commit was already tested on its branch/main + # push, so re-running here is redundant. See backwards-compatibility-test for why + # release builds must not run these gated tests. - name: Test migrations + if: ${{ !startsWith(github.ref, 'refs/tags/') }} run: ./docker/ci/test_migrations.sh + backwards-compatibility-test: + name: Backwards compatibility test + runs-on: ubuntu-24.04 + needs: [build] + permissions: + contents: read + + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Download digests + uses: actions/download-artifact@v8 + with: + path: ${{ runner.temp }}/digests + pattern: digests-amd64 + + # This sets the environment variable referenced in the docker-compose file + # for the image to use for the webapp container to the digest of the image + # that was built in the build job. + - name: Set webapp image + working-directory: ${{ runner.temp }}/digests/webapp + run: | + IMAGE="${{needs.build.outputs.webapp-repo}}$(printf '@sha256:%s' *)" + echo "$IMAGE" + echo "WEBAPP_IMAGE=$IMAGE" >> $GITHUB_ENV + + # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760 + - name: Disable network offload + run: sudo ethtool -K eth0 tx off rx off + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + # Skip on tag (release) builds. On a release the just-published tag becomes the "latest" + # release, so the script would resolve the previous-release image to this build's own + # X.Y.Z tag — which the push job hasn't created yet — deadlocking the release. + - name: Test backwards compatibility + if: ${{ !startsWith(github.ref, 'refs/tags/') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./docker/ci/test_backwards_compatibility.sh + push: name: Tag & Push Images runs-on: ubuntu-24.04 - needs: [build, integration-test, unit-test, migration-test] + needs: [build, integration-test, unit-test, migration-test, backwards-compatibility-test] permissions: contents: read packages: write diff --git a/README.md b/README.md index bcbb4f445c..16ed606758 100644 --- a/README.md +++ b/README.md @@ -781,10 +781,10 @@ Setting `PALACE_TEST_DATABASE_EXTERNAL_SCHEMA` to `true` tells the tests that th been applied to the provided database (by the current code), so the fixtures use that schema as-is, without dropping and recreating it from the current models. With database creation left enabled (the default), each worker clones the provided database as a template into its own per-worker database, so -the suite can still run in parallel. If you additionally set `PALACE_TEST_DATABASE_CREATE_DATABASE` to -`false`, the provided database is used directly, which requires the tests to be run serially. This is -used by the backwards-compatibility CI check, which runs a previous release's tests against a schema -built by the current code. +the suite can still run in parallel. This is how the backwards-compatibility CI check runs a previous +release's tests against a schema built by the current code. If you additionally set +`PALACE_TEST_DATABASE_CREATE_DATABASE` to `false`, the provided database is used directly, which +requires the tests to be run serially. ### Override `pytest` Arguments diff --git a/docker/ci/test_backwards_compatibility.sh b/docker/ci/test_backwards_compatibility.sh new file mode 100755 index 0000000000..af97359c92 --- /dev/null +++ b/docker/ci/test_backwards_compatibility.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# This script checks that the PREVIOUS released version of the application still works +# against the database schema produced by the CURRENT code. It exercises our online +# migration requirement: a release's database must keep working with version N-1 of the +# code, because during a deploy the new migrations run while the previous version's +# webservers are still serving traffic. +# +# It works by: +# (1) Finding the previous release (the latest GitHub release) and its published +# circ-webapp image. +# (2) Initializing a fresh database with the CURRENT image, which builds the new schema. +# (3) Running the previous release's database tests (pytest -m db) against that schema in +# external-schema mode (PALACE_TEST_DATABASE_EXTERNAL_SCHEMA), so the older code +# exercises the new schema. If those tests fail, the migration is not backwards +# compatible. +# +# The current image is provided via the WEBAPP_IMAGE environment variable (as in the other +# docker CI jobs). The previous release image can be overridden via PREV_RELEASE_IMAGE +# (useful for running this script locally). + +set -uo pipefail + +COMPOSE_FILES=(-f docker-compose.yml -f docker/ci/test_backwards_compatibility.yml) + +# Run a docker compose command for this check. +compose_cmd() { + docker compose "${COMPOSE_FILES[@]}" --progress quiet "$@" +} + +# Run a command in a container with the palace virtualenv activated. +run_in_container() { + local container="$1" + shift + compose_cmd run --rm --no-deps "${container}" /bin/bash -c "source env/bin/activate && $*" +} + +cleanup() { + compose_cmd down --remove-orphans >/dev/null 2>&1 +} + +fail() { + echo "::error::$1" + exit "${2:-1}" +} + +# (1) Resolve the previous release image. +if [[ -z "${PREV_RELEASE_IMAGE:-}" ]]; then + # In CI gh infers the repo from $GITHUB_REPOSITORY; locally it infers it from the git + # remote of the current directory. + gh_release_args=(release view --json tagName --jq '.tagName') + if [[ -n "${GITHUB_REPOSITORY:-}" ]]; then + gh_release_args+=(--repo "${GITHUB_REPOSITORY}") + fi + + # Resolve the previous release's tag. We deliberately do NOT fail the job when this lookup + # fails: `gh release view` exits non-zero both when there is genuinely no prior release (a + # legitimate skip) and on transient errors (auth hiccup, rate limit, network blip). But we + # don't fail *open silently*, we let gh's error flow to the job log and check its exit status, and on + # any failure we emit a GitHub Actions ::warning:: before skipping. That makes the gate no-op + # visible at the PR/checks level instead of quietly passing. + prev_tag="$(gh "${gh_release_args[@]}")" + gh_status=$? + if [[ ${gh_status} -ne 0 || -z "${prev_tag}" ]]; then + echo "::warning::Backwards-compatibility check skipped: could not resolve the previous release (gh exit ${gh_status}). The gate did NOT run for this build -- see gh's error above. A transient gh/auth/network failure or a genuine absence of releases both land here." + exit 0 + fi + + PREV_RELEASE_IMAGE="ghcr.io/thepalaceproject/circ-webapp:${prev_tag#v}" +fi +export PREV_RELEASE_IMAGE +echo "Previous release image: ${PREV_RELEASE_IMAGE}" + +if [[ -z "${WEBAPP_IMAGE:-}" ]]; then + fail "WEBAPP_IMAGE is not set; it must point at the current build's webapp image." +fi + +trap cleanup EXIT + +# (2) Build the current schema by initializing a fresh database with the current image. +compose_cmd up -d pg os minio redis || fail "Could not start service containers." +run_in_container webapp "./bin/util/initialize_instance" \ + || fail "Failed to initialize the database with the current image." + +# (3) Run the previous release's database tests against the new schema. In external-schema +# mode each xdist worker clones the freshly-built schema into its own database, so the suite +# runs in parallel (-n auto). +compose_cmd pull --quiet webapp-prev || fail "Could not pull ${PREV_RELEASE_IMAGE}." +echo "Running the previous release's database tests against the current schema ..." +if ! run_in_container webapp-prev \ + "uv sync --frozen --active && pytest --no-cov -n auto -m db --ignore=tests/migration tests"; then + fail "Previous release tests failed against the current schema: the migration is not backwards compatible." +fi + +echo "The previous release works against the current schema 🎉" diff --git a/docker/ci/test_backwards_compatibility.yml b/docker/ci/test_backwards_compatibility.yml new file mode 100644 index 0000000000..988fb44b0c --- /dev/null +++ b/docker/ci/test_backwards_compatibility.yml @@ -0,0 +1,20 @@ +# Compose overlay for the backwards-compatibility check; see test_backwards_compatibility.sh. +# +# The `webapp` service (from docker-compose.yml, the image built by the current commit) is +# used to build the new schema. `webapp-prev` runs a previously released image and re-runs +# its database tests against that schema in external-schema mode. +services: + webapp-prev: + # Inherit the full test environment (database/search/storage/redis URLs, etc.) from the + # webapp service so the previous release's test suite can run, then point it at the + # previous release's published image and turn on external-schema mode. + extends: + file: docker-compose.yml + service: webapp + image: "${PREV_RELEASE_IMAGE}" + environment: + PALACE_TEST_DATABASE_EXTERNAL_SCHEMA: "true" + # External-schema mode with database creation enabled clones the externally-built + # schema into a per-worker database, so the previous release's suite can run in + # parallel (-n auto); see DatabaseCreationFixture. + PALACE_TEST_DATABASE_CREATE_DATABASE: "true"