Add an "upgrade from previous" test

We have tests that do upgrades, but they start from the *new* bootc.
Add a `just`+CI workflow that starts from a stable shipped image
(we just need to inject tmt deps + nu).

The readonly test then gains a helper which optionally performs
an upgrade to the new target.

IOW the flow is

- deploy stock image with unmodified bootc etc
- upgrade
- run readonly tests

It could of course make sense to run *all* of the tests this
way as an optional thing, nothing blocks that, but it would
be *another* entry in our matrix and we're going to need
to figure out how to wrangle that matrix size.

Also do to that we'd need to abstract over TMT_REBOOT_COUNT.

Assisted-by: OpenCode (Claude claude-opus-4-6)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -262,6 +262,52 @@ jobs:
             ${{ env.ARCH }}"
           path: /var/tmp/tmt
 
+  # Test the upgrade path: boot from published base image, upgrade to locally-built image,
+  # then run readonly tests to verify the upgrade worked.
+  # Excluded: centos-9 (lacks systemd.extra-unit.* support needed for --bind-storage-ro)
+  test-upgrade:
+    needs: package
+    strategy:
+      fail-fast: false
+      matrix:
+        test_os: [fedora-43, centos-10]
+        variant: [ostree, composefs]
+
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Bootc Ubuntu Setup
+        uses: bootc-dev/actions/bootc-ubuntu-setup@main
+        with:
+          libvirt: true
+      - name: Install tmt
+        run: pip install --user "tmt[provision-virtual]"
+
+      - name: Setup env
+        run: |
+          BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
+          echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
+          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+          echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV
+          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV
+
+      - name: Download package artifacts
+        uses: actions/download-artifact@v8
+        with:
+          name: packages-${{ matrix.test_os }}
+          path: target/packages/
+
+      - name: Run upgrade test
+        run: just test-upgrade
+
+      - name: Archive TMT logs
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.variant }}-upgrade-${{ env.ARCH }}"
+          path: /var/tmp/tmt
+
   # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues
   # when run in the same job as test-integration).
   # Uses fedora-43 as it's the current stable Fedora release matching CoreOS.
@@ -342,7 +388,7 @@ jobs:
   # Sentinel job for required checks - configure this job name in repository settings
   required-checks:
     if: always()
-    needs: [cargo-deny, validate, package, test-integration, test-coreos, test-container-export]
+    needs: [cargo-deny, validate, package, test-integration, test-upgrade, test-coreos, test-container-export]
     runs-on: ubuntu-latest
     steps:
       - run: exit 1
@@ -351,5 +397,6 @@ jobs:
           needs.validate.result != 'success' ||
           needs.package.result != 'success' ||
           needs.test-integration.result != 'success' ||
+          needs.test-upgrade.result != 'success' ||
           needs.test-coreos.result != 'success' ||
           needs.test-container-export.result != 'success'

Justfile

@@ -17,6 +17,8 @@
 base_img := "localhost/bootc"
 # Synthetic upgrade image for testing
 upgrade_img := base_img + "-upgrade"
+# Base image with tmt dependencies added, used as the boot source for upgrade tests
+upgrade_source_img := base_img + "-upgrade-source"
 
 # Build variant: ostree (default) or composefs
 variant := env("BOOTC_variant", "ostree")
@@ -141,6 +143,29 @@ test-composefs bootloader filesystem boot_type seal_state *ARGS:
                 {{ARGS}} \
                 $(if [ "{{boot_type}}" = "uki" ]; then echo "readonly"; else echo "integration"; fi)
 
+# Run upgrade test: boot VM from published base image (with tmt deps added),
+# upgrade to locally-built image, reboot, then run readonly tests to verify.
+# The --upgrade-image flag triggers --bind-storage-ro in bcvk, making the
+# locally-built image available inside the VM via containers-storage transport.
+[group('core')]
+test-upgrade *ARGS: build _build-upgrade-source-image
+    #!/bin/bash
+    set -xeuo pipefail
+    composefs_args=()
+    if [[ "{{variant}}" = composefs ]]; then
+        composefs_args=(--composefs-backend \
+            --bootloader={{bootloader}} \
+            --filesystem={{filesystem}} \
+            --seal-state={{seal_state}} \
+            --boot-type={{boot_type}} \
+            --karg=enforcing=0)
+    fi
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} \
+        --env=BOOTC_test_upgrade_image={{base_img}} \
+        --upgrade-image={{base_img}} \
+        "${composefs_args[@]}" \
+        {{upgrade_source_img}} {{ARGS}} readonly
+
 # Run cargo fmt and clippy checks in container
 [group('core')]
 validate:
@@ -339,6 +364,10 @@ _keygen:
 _build-upgrade-image:
     cat tmt/tests/Dockerfile.upgrade | podman build -t {{upgrade_img}} --from={{base_img}} -
 
+# Build the upgrade source image: base image + tmt dependencies (rsync, nu, cloud-init)
+_build-upgrade-source-image:
+    podman build --build-arg=base={{base}} --build-arg=variant={{variant}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source .
+
 # Copy an image from user podman storage to root's podman storage
 # This allows building as regular user then running privileged tests
 [group('testing')]

crates/xtask/src/tmt.rs

@@ -2,7 +2,7 @@ use anyhow::{Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
 use fn_error_context::context;
 use rand::RngExt;
-use xshell::{Shell, cmd};
+use xshell::{cmd, Shell};
 
 // Generation markers for integration.fmf
 const PLAN_MARKER_BEGIN: &str = "# BEGIN GENERATED PLANS\n";
@@ -492,6 +492,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
                 opts.push(format!("--bootloader={b}"));
             }
 
+            for k in &args.karg {
+                opts.push(format!("--karg={k}"));
+            }
+
             opts
         };
 

crates/xtask/src/xtask.rs

@@ -14,7 +14,7 @@ use anyhow::{Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
 use clap::{Args, Parser, Subcommand, ValueEnum};
 use fn_error_context::context;
-use xshell::{Shell, cmd};
+use xshell::{cmd, Shell};
 
 mod buildsys;
 mod man;
@@ -176,6 +176,10 @@ pub(crate) struct RunTmtArgs {
     // Required to send kargs to only bls installs
     #[arg(long, default_value_t, requires = "composefs_backend")]
     pub(crate) boot_type: BootType,
+
+    /// Additional kernel arguments to pass to bcvk
+    #[arg(long)]
+    pub(crate) karg: Vec<String>,
 }
 
 /// Arguments for tmt-provision command

tmt/tests/Dockerfile.upgrade-source

@@ -0,0 +1,32 @@
+# Build an image suitable for upgrade testing: takes the published base image
+# and adds the minimal dependencies needed by tmt (rsync, nu, etc.)
+# so we can boot it in a VM, then upgrade into the locally-built image.
+#
+# Note: we do NOT pass `cloudinit` to provision-derived.sh because bcvk
+# handles SSH key injection itself; cloud-init interferes with that.
+ARG base
+FROM ${base}
+COPY hack/provision-derived.sh hack/packages.txt contrib/packaging/configure-rootfs /run/provision/
+ARG variant=ostree
+RUN --mount=type=tmpfs,target=/tmp <<EORUN
+set -xeuo pipefail
+cd /run/provision
+# Install nu and tmt dependencies using the same script as the main build,
+# but with SKIP_CONFIGS=1 since we don't need LBIs or test kargs in the
+# upgrade source image (those come from the image we upgrade into).
+SKIP_CONFIGS=1 ./provision-derived.sh
+# Ensure a root filesystem type is configured so `bootc install to-disk`
+# works for base images that don't ship a default (e.g. Fedora).
+# For the ostree variant, configure-rootfs will default to xfs.
+./configure-rootfs "${variant}"
+# For composefs, rebuild the initramfs to include the bootc dracut module.
+# The module's check() returns 255 so it's never auto-included, but it's
+# required for composefs root setup during boot.
+if [[ "${variant}" == composefs* ]]; then
+    mkdir -p /etc/dracut.conf.d
+    echo 'add_dracutmodules+=" bootc "' > /etc/dracut.conf.d/50-bootc-composefs.conf
+    kver=$(cd /usr/lib/modules && echo *)
+    dracut --force --kver "$kver" "/usr/lib/modules/$kver/initramfs.img"
+fi
+rm -rf /run/provision
+EORUN

tmt/tests/booted/bootc_testlib.nu

@@ -1,4 +1,4 @@
-# A simple nushell "library" for the
+# A simple nushell "library" for bootc test helpers
 
 # This is a workaround for what must be a systemd bug
 # that seems to have appeared in C10S
@@ -23,3 +23,56 @@ export def have_hostexports [] {
 export def parse_cmdline []  {
     open /proc/cmdline | str trim | split row " "
 }
+
+# If the BOOTC_test_upgrade_image environment variable is set, performs
+# an upgrade to that image and reboots on the first boot. On the second
+# boot (after the upgrade), verifies we're running the upgraded image
+# and returns so the caller can proceed with its tests.
+#
+# This enables an "upgrade test" flow: boot from a published base image,
+# upgrade to the image under test, reboot, then run verification tests.
+#
+# Note: This uses BOOTC_test_upgrade_image (the image to upgrade *into*),
+# which is distinct from BOOTC_upgrade_image (the synthetic upgrade image
+# used by existing upgrade tests like test-image-upgrade-reboot).
+#
+# Returns without doing anything if BOOTC_test_upgrade_image is not set.
+export def maybe_upgrade [] {
+    use std assert
+
+    let upgrade_image = $env.BOOTC_test_upgrade_image? | default ""
+    if $upgrade_image == "" {
+        return
+    }
+
+    match $env.TMT_REBOOT_COUNT? {
+        null | "0" => {
+            if not (have_hostexports) {
+                error make { msg: "BOOTC_test_upgrade_image is set but host exports (--bind-storage-ro) are not available" }
+            }
+            # Save the pre-upgrade bootc version so post-upgrade tests
+            # can detect known incompatibilities with older versions.
+            let pre_ver = (bootc --version | parse "bootc {v}" | get 0.v)
+            $pre_ver | save /var/bootc-pre-upgrade-version
+            print $"Pre-upgrade bootc version: ($pre_ver)"
+
+            print $"Upgrade image specified: ($upgrade_image)"
+            print "Performing upgrade switch..."
+            bootc switch --transport containers-storage $upgrade_image
+            print "Switch complete, rebooting..."
+            tmt-reboot
+        },
+        "1" => {
+            print $"Second boot after upgrade to ($upgrade_image)"
+            let st = bootc status --json | from json
+            let booted = $st.status.booted.image
+            assert equal $booted.image.transport "containers-storage"
+            assert equal $booted.image.image $upgrade_image
+            print "Upgrade verified, continuing with tests..."
+        },
+        $o => {
+            # For higher reboot counts, just continue - the caller
+            # may have its own reboot logic
+        },
+    }
+}

tmt/tests/booted/readonly/031-test-upgrade-check.nu

@@ -0,0 +1,41 @@
+use std assert
+use tap.nu
+
+tap begin "verify bootc upgrade --check succeeds after upgrade"
+
+# After an upgrade (bootc switch), verify that the running system can
+# still query for further upgrades. This catches regressions where
+# on-disk state created by an older bootc is incompatible with the
+# new bootc's upgrade machinery (e.g. #2074).
+# Only meaningful when we actually performed an upgrade.
+let upgrade_image = $env.BOOTC_test_upgrade_image? | default ""
+if $upgrade_image == "" {
+    print "# skip: not an upgrade test (BOOTC_test_upgrade_image not set)"
+    tap ok
+    exit 0
+}
+
+# Read the pre-upgrade bootc version saved during first boot.
+let old_version_str = (open /var/bootc-pre-upgrade-version | str trim)
+let old_ver = ($old_version_str | split row "." | each { into int })
+print $"Pre-upgrade bootc version: ($old_version_str)"
+
+let is_composefs = (tap is_composefs)
+
+print "Running bootc upgrade --check to verify upgrade machinery works..."
+let result = do -i { bootc upgrade --check } | complete
+if $result.exit_code == 0 {
+    print "bootc upgrade --check succeeded"
+} else {
+    print $"bootc upgrade --check failed: ($result.stderr)"
+    # Known failure: composefs upgrades from bootc <= 1.13 have
+    # incompatible on-disk state (see #2074).
+    let old_bootc_le_1_13 = ($old_ver.0 < 1) or (($old_ver.0 == 1) and ($old_ver.1 <= 13))
+    if $is_composefs and $old_bootc_le_1_13 {
+        print $"# known failure: composefs upgrade --check from bootc ($old_version_str) \(see #2074\)"
+    } else {
+        error make { msg: $"bootc upgrade --check failed unexpectedly: ($result.stderr)" }
+    }
+}
+
+tap ok

tmt/tests/booted/test-01-readonly.nu

@@ -7,9 +7,15 @@
 #
 # Run all readonly tests in sequence
 use tap.nu
+use bootc_testlib.nu
 
 tap begin "readonly tests"
 
+# If an upgrade image is specified (via BOOTC_test_upgrade_image env var),
+# perform the upgrade and reboot first. On the second boot after upgrade,
+# this returns and we continue with the readonly tests below.
+bootc_testlib maybe_upgrade
+
 # Get all readonly test files and run them in order
 let tests = (ls booted/readonly/*-test-*.nu | get name | sort)