Add an "upgrade from previous" test

We have tests that do upgrades, but they start from the *new* bootc.
Add a `just`+CI workflow that starts from a stable shipped image
(we just need to inject tmt deps + nu).

The readonly test then gains a helper which optionally performs
an upgrade to the new target.

IOW the flow is

- deploy stock image with unmodified bootc etc
- upgrade
- run readonly tests

It could of course make sense to run *all* of the tests this
way as an optional thing, nothing blocks that, but it would
be *another* entry in our matrix and we're going to need
to figure out how to wrangle that matrix size.

Also do to that we'd need to abstract over TMT_REBOOT_COUNT.

Assisted-by: OpenCode (Claude claude-opus-4-6)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -261,6 +261,50 @@ jobs:
             ${{ env.ARCH }}"
           path: /var/tmp/tmt
 
+  # Test the upgrade path: boot from published base image, upgrade to locally-built image,
+  # then run readonly tests to verify the upgrade worked.
+  # Excluded: centos-9 (lacks systemd.extra-unit.* support needed for --bind-storage-ro)
+  test-upgrade:
+    needs: package
+    strategy:
+      fail-fast: false
+      matrix:
+        test_os: [fedora-43, centos-10]
+
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Bootc Ubuntu Setup
+        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+        with:
+          libvirt: true
+      - name: Install tmt
+        run: pip install --user "tmt[provision-virtual]"
+
+      - name: Setup env
+        run: |
+          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
+          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
+          echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV
+          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV
+
+      - name: Download package artifacts
+        uses: actions/download-artifact@v8
+        with:
+          name: packages-${{ matrix.test_os }}
+          path: target/packages/
+
+      - name: Run upgrade test
+        run: just test-upgrade
+
+      - name: Archive TMT logs
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-upgrade-${{ env.ARCH }}"
+          path: /var/tmp/tmt
+
   # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues
   # when run in the same job as test-integration).
   # Uses fedora-43 as it's the current stable Fedora release matching CoreOS.
@@ -341,7 +385,7 @@ jobs:
   # Sentinel job for required checks - configure this job name in repository settings
   required-checks:
     if: always()
-    needs: [cargo-deny, validate, package, test-integration, test-coreos, test-container-export]
+    needs: [cargo-deny, validate, package, test-integration, test-upgrade, test-coreos, test-container-export]
     runs-on: ubuntu-latest
     steps:
       - run: exit 1
@@ -350,5 +394,6 @@ jobs:
           needs.validate.result != 'success' ||
           needs.package.result != 'success' ||
           needs.test-integration.result != 'success' ||
+          needs.test-upgrade.result != 'success' ||
           needs.test-coreos.result != 'success' ||
           needs.test-container-export.result != 'success'

Justfile

@@ -17,6 +17,8 @@
 base_img := "localhost/bootc"
 # Synthetic upgrade image for testing
 upgrade_img := base_img + "-upgrade"
+# Base image with tmt dependencies added, used as the boot source for upgrade tests
+upgrade_source_img := base_img + "-upgrade-source"
 
 # Build variant: ostree (default) or composefs
 variant := env("BOOTC_variant", "ostree")
@@ -141,6 +143,14 @@ test-composefs bootloader filesystem boot_type seal_state *ARGS:
                 {{ARGS}} \
                 $(if [ "{{boot_type}}" = "uki" ]; then echo "readonly"; else echo "integration"; fi)
 
+# Run upgrade test: boot VM from published base image (with tmt deps added),
+# upgrade to locally-built image, reboot, then run readonly tests to verify.
+# The --upgrade-image flag triggers --bind-storage-ro in bcvk, making the
+# locally-built image available inside the VM via containers-storage transport.
+[group('core')]
+test-upgrade *ARGS: build _build-upgrade-source-image
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_test_upgrade_image={{base_img}} --upgrade-image={{base_img}} {{upgrade_source_img}} {{ARGS}} readonly
+
 # Run cargo fmt and clippy checks in container
 [group('core')]
 validate:
@@ -339,6 +349,10 @@ _keygen:
 _build-upgrade-image:
     cat tmt/tests/Dockerfile.upgrade | podman build -t {{upgrade_img}} --from={{base_img}} -
 
+# Build the upgrade source image: base image + tmt dependencies (rsync, nu, cloud-init)
+_build-upgrade-source-image:
+    podman build --build-arg=base={{base}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source .
+
 # Copy an image from user podman storage to root's podman storage
 # This allows building as regular user then running privileged tests
 [group('testing')]

tmt/tests/Dockerfile.upgrade-source

@@ -0,0 +1,18 @@
+# Build an image suitable for upgrade testing: takes the published base image
+# and adds the minimal dependencies needed by tmt (rsync, nu, etc.)
+# so we can boot it in a VM, then upgrade into the locally-built image.
+#
+# Note: we do NOT pass `cloudinit` to provision-derived.sh because bcvk
+# handles SSH key injection itself; cloud-init interferes with that.
+ARG base
+FROM ${base}
+COPY hack/provision-derived.sh hack/packages.txt /run/provision/
+RUN --mount=type=tmpfs,target=/tmp <<EORUN
+set -xeuo pipefail
+cd /run/provision
+# Install nu and tmt dependencies using the same script as the main build,
+# but with SKIP_CONFIGS=1 since we don't need LBIs or test kargs in the
+# upgrade source image (those come from the image we upgrade into).
+SKIP_CONFIGS=1 ./provision-derived.sh
+rm -rf /run/provision
+EORUN

tmt/tests/booted/bootc_testlib.nu

@@ -1,4 +1,4 @@
-# A simple nushell "library" for the
+# A simple nushell "library" for bootc test helpers
 
 # This is a workaround for what must be a systemd bug
 # that seems to have appeared in C10S
@@ -23,3 +23,50 @@ export def have_hostexports [] {
 export def parse_cmdline []  {
     open /proc/cmdline | str trim | split row " "
 }
+
+# If the BOOTC_test_upgrade_image environment variable is set, performs
+# an upgrade to that image and reboots on the first boot. On the second
+# boot (after the upgrade), verifies we're running the upgraded image
+# and returns so the caller can proceed with its tests.
+#
+# This enables an "upgrade test" flow: boot from a published base image,
+# upgrade to the image under test, reboot, then run verification tests.
+#
+# Note: This uses BOOTC_test_upgrade_image (the image to upgrade *into*),
+# which is distinct from BOOTC_upgrade_image (the synthetic upgrade image
+# used by existing upgrade tests like test-image-upgrade-reboot).
+#
+# Returns without doing anything if BOOTC_test_upgrade_image is not set.
+export def maybe_upgrade [] {
+    use std assert
+
+    let upgrade_image = $env.BOOTC_test_upgrade_image? | default ""
+    if $upgrade_image == "" {
+        return
+    }
+
+    match $env.TMT_REBOOT_COUNT? {
+        null | "0" => {
+            if not (have_hostexports) {
+                error make { msg: "BOOTC_test_upgrade_image is set but host exports (--bind-storage-ro) are not available" }
+            }
+            print $"Upgrade image specified: ($upgrade_image)"
+            print "Performing upgrade switch..."
+            bootc switch --transport containers-storage $upgrade_image
+            print "Switch complete, rebooting..."
+            tmt-reboot
+        },
+        "1" => {
+            print $"Second boot after upgrade to ($upgrade_image)"
+            let st = bootc status --json | from json
+            let booted = $st.status.booted.image
+            assert equal $booted.image.transport "containers-storage"
+            assert equal $booted.image.image $upgrade_image
+            print "Upgrade verified, continuing with tests..."
+        },
+        $o => {
+            # For higher reboot counts, just continue - the caller
+            # may have its own reboot logic
+        },
+    }
+}

tmt/tests/booted/test-01-readonly.nu

@@ -7,9 +7,15 @@
 #
 # Run all readonly tests in sequence
 use tap.nu
+use bootc_testlib.nu
 
 tap begin "readonly tests"
 
+# If an upgrade image is specified (via BOOTC_test_upgrade_image env var),
+# perform the upgrade and reboot first. On the second boot after upgrade,
+# this returns and we continue with the readonly tests below.
+bootc_testlib maybe_upgrade
+
 # Get all readonly test files and run them in order
 let tests = (ls booted/readonly/*-test-*.nu | get name | sort)