ci/upgrade: Add composefs variant and upgrade --check test

Extend the upgrade test matrix with an ostree/composefs variant
dimension, so we test upgrading both storage backends.

For composefs, the upgrade source image needs:
- The bootc dracut module in the initramfs (auto-excluded by check())
- A fsverity-capable root filesystem (ext4 or btrfs via configure-rootfs)
- enforcing=0 karg (base image SELinux policy lacks composefs contexts)

The enforcing=0 karg is also added to the target image test-kargs so
the post-upgrade boot works.

A new readonly test (031-test-upgrade-check) runs `bootc upgrade --check`
after the upgrade to verify the running system can still self-update.
For composefs with pre-upgrade bootc <= 1.13, this is a known failure
(#2074) due to incompatible on-disk state. The pre-upgrade bootc version
is saved to /var/bootc-pre-upgrade-version during first boot.

Assisted-by: OpenCode (Claude claude-opus-4-6)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -270,6 +270,7 @@ jobs:
       fail-fast: false
       matrix:
         test_os: [fedora-43, centos-10]
+        variant: [ostree, composefs]
 
     runs-on: ubuntu-24.04
 
@@ -286,6 +287,7 @@ jobs:
         run: |
           BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
           echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
+          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
           echo "BOOTC_SKIP_PACKAGE=1" >> $GITHUB_ENV
           echo "RUST_BACKTRACE=full" >> $GITHUB_ENV
 
@@ -302,7 +304,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-upgrade-${{ env.ARCH }}"
+          name: "tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.variant }}-upgrade-${{ env.ARCH }}"
           path: /var/tmp/tmt
 
   # Test bootc install on Fedora CoreOS (separate job to avoid disk space issues

Justfile

@@ -149,7 +149,22 @@ test-composefs bootloader filesystem boot_type seal_state *ARGS:
 # locally-built image available inside the VM via containers-storage transport.
 [group('core')]
 test-upgrade *ARGS: build _build-upgrade-source-image
-    cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_test_upgrade_image={{base_img}} --upgrade-image={{base_img}} {{upgrade_source_img}} {{ARGS}} readonly
+    #!/bin/bash
+    set -xeuo pipefail
+    composefs_args=()
+    if [[ "{{variant}}" = composefs ]]; then
+        composefs_args=(--composefs-backend \
+            --bootloader={{bootloader}} \
+            --filesystem={{filesystem}} \
+            --seal-state={{seal_state}} \
+            --boot-type={{boot_type}} \
+            --karg=enforcing=0)
+    fi
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} \
+        --env=BOOTC_test_upgrade_image={{base_img}} \
+        --upgrade-image={{base_img}} \
+        "${composefs_args[@]}" \
+        {{upgrade_source_img}} {{ARGS}} readonly
 
 # Run cargo fmt and clippy checks in container
 [group('core')]
@@ -351,7 +366,7 @@ _build-upgrade-image:
 
 # Build the upgrade source image: base image + tmt dependencies (rsync, nu, cloud-init)
 _build-upgrade-source-image:
-    podman build --build-arg=base={{base}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source .
+    podman build --build-arg=base={{base}} --build-arg=variant={{variant}} -t {{upgrade_source_img}} -f tmt/tests/Dockerfile.upgrade-source .
 
 # Copy an image from user podman storage to root's podman storage
 # This allows building as regular user then running privileged tests

crates/xtask/src/tmt.rs

@@ -492,6 +492,10 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
                 opts.push(format!("--bootloader={b}"));
             }
 
+            for k in &args.karg {
+                opts.push(format!("--karg={k}"));
+            }
+
             opts
         };
 

crates/xtask/src/xtask.rs

@@ -178,6 +178,10 @@ pub(crate) struct RunTmtArgs {
     // Required to send kargs to only bls installs
     #[arg(long, default_value_t, requires = "composefs_backend")]
     pub(crate) boot_type: BootType,
+
+    /// Additional kernel arguments to pass to bcvk
+    #[arg(long)]
+    pub(crate) karg: Vec<String>,
 }
 
 /// Arguments for tmt-provision command

tmt/tests/Dockerfile.upgrade-source

@@ -6,13 +6,27 @@
 # handles SSH key injection itself; cloud-init interferes with that.
 ARG base
 FROM ${base}
-COPY hack/provision-derived.sh hack/packages.txt /run/provision/
+COPY hack/provision-derived.sh hack/packages.txt contrib/packaging/configure-rootfs /run/provision/
+ARG variant=ostree
 RUN --mount=type=tmpfs,target=/tmp <<EORUN
 set -xeuo pipefail
 cd /run/provision
 # Install nu and tmt dependencies using the same script as the main build,
 # but with SKIP_CONFIGS=1 since we don't need LBIs or test kargs in the
 # upgrade source image (those come from the image we upgrade into).
 SKIP_CONFIGS=1 ./provision-derived.sh
+# Ensure a root filesystem type is configured so `bootc install to-disk`
+# works for base images that don't ship a default (e.g. Fedora).
+# For the ostree variant, configure-rootfs will default to xfs.
+./configure-rootfs "${variant}"
+# For composefs, rebuild the initramfs to include the bootc dracut module.
+# The module's check() returns 255 so it's never auto-included, but it's
+# required for composefs root setup during boot.
+if [[ "${variant}" == composefs* ]]; then
+    mkdir -p /etc/dracut.conf.d
+    echo 'add_dracutmodules+=" bootc "' > /etc/dracut.conf.d/50-bootc-composefs.conf
+    kver=$(cd /usr/lib/modules && echo *)
+    dracut --force --kver "$kver" "/usr/lib/modules/$kver/initramfs.img"
+fi
 rm -rf /run/provision
 EORUN

tmt/tests/booted/bootc_testlib.nu

@@ -50,6 +50,12 @@ export def maybe_upgrade [] {
             if not (have_hostexports) {
                 error make { msg: "BOOTC_test_upgrade_image is set but host exports (--bind-storage-ro) are not available" }
             }
+            # Save the pre-upgrade bootc version so post-upgrade tests
+            # can detect known incompatibilities with older versions.
+            let pre_ver = (bootc --version | parse "bootc {v}" | get 0.v)
+            $pre_ver | save /var/bootc-pre-upgrade-version
+            print $"Pre-upgrade bootc version: ($pre_ver)"
+
             print $"Upgrade image specified: ($upgrade_image)"
             print "Performing upgrade switch..."
             bootc switch --transport containers-storage $upgrade_image

tmt/tests/booted/readonly/031-test-upgrade-check.nu

@@ -0,0 +1,41 @@
+use std assert
+use tap.nu
+
+tap begin "verify bootc upgrade --check succeeds after upgrade"
+
+# After an upgrade (bootc switch), verify that the running system can
+# still query for further upgrades. This catches regressions where
+# on-disk state created by an older bootc is incompatible with the
+# new bootc's upgrade machinery (e.g. #2074).
+# Only meaningful when we actually performed an upgrade.
+let upgrade_image = $env.BOOTC_test_upgrade_image? | default ""
+if $upgrade_image == "" {
+    print "# skip: not an upgrade test (BOOTC_test_upgrade_image not set)"
+    tap ok
+    exit 0
+}
+
+# Read the pre-upgrade bootc version saved during first boot.
+let old_version_str = (open /var/bootc-pre-upgrade-version | str trim)
+let old_ver = ($old_version_str | split row "." | each { into int })
+print $"Pre-upgrade bootc version: ($old_version_str)"
+
+let is_composefs = (tap is_composefs)
+
+print "Running bootc upgrade --check to verify upgrade machinery works..."
+let result = do -i { bootc upgrade --check } | complete
+if $result.exit_code == 0 {
+    print "bootc upgrade --check succeeded"
+} else {
+    print $"bootc upgrade --check failed: ($result.stderr)"
+    # Known failure: composefs upgrades from bootc <= 1.13 have
+    # incompatible on-disk state (see #2074).
+    let old_bootc_le_1_13 = ($old_ver.0 < 1) or (($old_ver.0 == 1) and ($old_ver.1 <= 13))
+    if $is_composefs and $old_bootc_le_1_13 {
+        print $"# known failure: composefs upgrade --check from bootc ($old_version_str) \(see #2074\)"
+    } else {
+        error make { msg: $"bootc upgrade --check failed unexpectedly: ($result.stderr)" }
+    }
+}
+
+tap ok