feat: add upgrade/switch/rollback e2e tests for all 4 boot variants

.github/workflows/e2e.yml

@@ -130,6 +130,23 @@ jobs:
       - name: Run e2e tests (UKI+SB)
         run: just e2e-uki-secureboot disk-uki-sb.raw
 
+      - name: Pull registry image for upgrade tests
+        run: sudo podman pull docker.io/library/registry:2
+
+      - name: Run upgrade/switch/rollback tests (GRUB)
+        run: just e2e-upgrade composefs-os-test:latest disk.raw
+
+      - name: Run upgrade/switch/rollback tests (GRUB + Secure Boot)
+        run: just e2e-upgrade-secureboot composefs-os-test:latest disk-sb.raw
+
+      - name: Run upgrade/switch/rollback tests (UKI)
+        run: |
+          OVMF_VARS=$(find /usr/share -name 'OVMF_VARS.fd' ! -name '*.secboot*' ! -name '*.ms.*' | head -1)
+          just e2e-uki-upgrade composefs-os-uki-test:latest disk-uki.raw "$OVMF_VARS"
+
+      - name: Run upgrade/switch/rollback tests (UKI + Secure Boot)
+        run: just e2e-uki-upgrade-secureboot composefs-os-uki-sb-test:latest disk-uki-sb.raw
+
       - name: Upload disk images on failure
         if: failure()
         uses: actions/upload-artifact@v4
@@ -260,6 +277,23 @@ jobs:
       - name: Run e2e tests (Ubuntu UKI+SB)
         run: just e2e-uki-secureboot disk-ubuntu-uki-sb.raw
 
+      - name: Pull registry image for upgrade tests
+        run: sudo podman pull docker.io/library/registry:2
+
+      - name: Run upgrade/switch/rollback tests (Ubuntu GRUB)
+        run: just e2e-upgrade-ubuntu composefs-os-ubuntu-test:latest disk-ubuntu.raw
+
+      - name: Run upgrade/switch/rollback tests (Ubuntu GRUB + Secure Boot)
+        run: just e2e-upgrade-secureboot-ubuntu composefs-os-ubuntu-test:latest disk-ubuntu-sb.raw
+
+      - name: Run upgrade/switch/rollback tests (Ubuntu UKI)
+        run: |
+          OVMF_VARS=$(find /usr/share -name 'OVMF_VARS.fd' ! -name '*.secboot*' ! -name '*.ms.*' | head -1)
+          just e2e-uki-upgrade-ubuntu composefs-os-ubuntu-uki-test:latest disk-ubuntu-uki.raw "$OVMF_VARS"
+
+      - name: Run upgrade/switch/rollback tests (Ubuntu UKI + Secure Boot)
+        run: just e2e-uki-upgrade-secureboot-ubuntu composefs-os-ubuntu-uki-sb-test:latest disk-ubuntu-uki-sb.raw
+
       - name: Upload Ubuntu disk images on failure
         if: failure()
         uses: actions/upload-artifact@v4

.gitignore

@@ -4,3 +4,5 @@
 *.raw
 *.raw.sb.cer
 ovmf-vars-*.fd
+/tests/__pycache__/
+/e2e-console.log

Containerfile.ubuntu

@@ -73,7 +73,11 @@ RUN mkdir -p /usr/share/efi/EFI/ubuntu && \
 # ---------------------------------------------------------------------------
 FROM rootfs-base AS rootfs-uki
 
+# systemd-boot: provides bootctl (boot entry management, needed by cbootc at runtime).
+# systemd-boot-efi: provides the systemd-bootx64.efi binary copied to the ESP by cbootc install.
+# Ubuntu splits these into two packages; on Fedora both are in systemd-boot-unsigned.
 RUN apt-get update && apt-get install -y --no-install-recommends \
+        systemd-boot \
         systemd-boot-efi \
         systemd-ukify \
     && rm -rf /var/lib/apt/lists/*
@@ -83,7 +87,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # ---------------------------------------------------------------------------
 FROM rootfs-base AS rootfs-uki-sb
 
+# Same rationale as rootfs-uki above.
 RUN apt-get update && apt-get install -y --no-install-recommends \
+        systemd-boot \
         systemd-boot-efi \
         systemd-ukify \
         sbsigntool \

examples/fedora/Containerfile

@@ -34,8 +34,9 @@ RUN mkdir -p /usr/lib/systemd/system/serial-getty@ttyS0.service.d && \
     printf '[Service]\nExecStart=\nExecStart=-/sbin/agetty --autologin root --noclear %%I 115200 vt220\n' \
         > /usr/lib/systemd/system/serial-getty@ttyS0.service.d/autologin.conf
 
+# gawk provides 'awk', used by the e2e test's configure_guest_network helper.
 # Add your packages here, e.g.:
-RUN dnf install -y --setopt=install_weak_deps=False vim htop && dnf clean all
+RUN dnf install -y --setopt=install_weak_deps=False gawk && dnf clean all
 
 # --- Do not edit below this line ------------------------------------------
 

examples/ubuntu/Containerfile

@@ -34,9 +34,10 @@ RUN mkdir -p /usr/lib/systemd/system/serial-getty@ttyS0.service.d && \
     printf '[Service]\nExecStart=\nExecStart=-/sbin/agetty --autologin root --noclear %%I 115200 vt220\n' \
         > /usr/lib/systemd/system/serial-getty@ttyS0.service.d/autologin.conf
 
+# gawk provides 'awk', used by the e2e test's configure_guest_network helper.
 # Add your packages here, e.g.:
-# RUN apt-get install -y --no-install-recommends vim htop \
-#     && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends gawk \
+    && rm -rf /var/lib/apt/lists/*
 
 # --- Do not edit below this line ------------------------------------------
 

justfile

@@ -176,6 +176,51 @@ e2e-uki-secureboot disk="disk-uki-sb.raw" ovmf_vars="":
     fi
     python3 tests/e2e.py --uki-secureboot --ovmf-vars "$vars" {{disk}}
 
+# ── Upgrade / Switch / Rollback e2e tests ────────────────────────────────────
+
+# Run upgrade/switch/rollback e2e against a GRUB disk image (requires sudo for podman)
+e2e-upgrade image=example_image disk="disk.raw":
+    sudo python3 tests/e2e.py --upgrade --source-image {{image}} {{disk}}
+
+# Run upgrade/switch/rollback e2e against a GRUB + Secure Boot disk image
+e2e-upgrade-secureboot image=example_image disk="disk-sb.raw":
+    sudo python3 tests/e2e.py --upgrade --secure-boot --source-image {{image}} {{disk}}
+
+# Run upgrade/switch/rollback e2e against a UKI/systemd-boot disk image
+e2e-uki-upgrade image=example_image_uki disk="disk-uki.raw" ovmf_vars="":
+    v="{{ovmf_vars}}"; sudo python3 tests/e2e.py --upgrade --uki --source-image {{image}} ${v:+--ovmf-vars "$v"} {{disk}}
+
+# Run upgrade/switch/rollback e2e against a UKI + Secure Boot disk image
+e2e-uki-upgrade-secureboot image=example_image_uki_sb disk="disk-uki-sb.raw" ovmf_vars="":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    vars="{{ovmf_vars}}"
+    if [ -z "$vars" ]; then
+        just prep-sb-vars {{disk}} ovmf-vars-uki-sb.fd
+        vars=ovmf-vars-uki-sb.fd
+    fi
+    sudo python3 tests/e2e.py --upgrade --uki-secureboot --source-image {{image}} --ovmf-vars "$vars" {{disk}}
+
+# Ubuntu upgrade variants
+e2e-upgrade-ubuntu image=example_image_ubuntu disk="disk-ubuntu.raw":
+    sudo python3 tests/e2e.py --upgrade --source-image {{image}} {{disk}}
+
+e2e-upgrade-secureboot-ubuntu image=example_image_ubuntu disk="disk-ubuntu-sb.raw":
+    sudo python3 tests/e2e.py --upgrade --secure-boot --source-image {{image}} {{disk}}
+
+e2e-uki-upgrade-ubuntu image=example_image_ubuntu_uki disk="disk-ubuntu-uki.raw" ovmf_vars="":
+    v="{{ovmf_vars}}"; sudo python3 tests/e2e.py --upgrade --uki --source-image {{image}} ${v:+--ovmf-vars "$v"} {{disk}}
+
+e2e-uki-upgrade-secureboot-ubuntu image=example_image_ubuntu_uki_sb disk="disk-ubuntu-uki-sb.raw" ovmf_vars="":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    vars="{{ovmf_vars}}"
+    if [ -z "$vars" ]; then
+        just prep-sb-vars {{disk}} ovmf-vars-ubuntu-uki-sb.fd
+        vars=ovmf-vars-ubuntu-uki-sb.fd
+    fi
+    sudo python3 tests/e2e.py --upgrade --uki-secureboot --source-image {{image}} --ovmf-vars "$vars" {{disk}}
+
 # ── Convenience combos ────────────────────────────────────────────────────────
 
 # Rust checks only — fast, no containers needed
@@ -216,6 +261,16 @@ ci-ubuntu-uki-secureboot: build-base-ubuntu-uki-secureboot (build-example-ubuntu
     just install-disk-uki-secureboot {{example_image_ubuntu_uki_sb}} disk-ubuntu-uki-sb.raw 5G
     just e2e-uki-secureboot disk-ubuntu-uki-sb.raw
 
+# Full GRUB upgrade/switch/rollback workflow (Fedora)
+ci-grub-upgrade: build-base (build-example base_image)
+    just install-disk
+    just e2e-upgrade
+
+# Full UKI upgrade/switch/rollback workflow (Fedora)
+ci-uki-upgrade: build-base-uki (build-example-uki base_image_uki)
+    just install-disk-uki
+    just e2e-uki-upgrade
+
 # ── Cleanup ───────────────────────────────────────────────────────────────────
 
 # Remove Rust build artifacts

src/rollback.rs

@@ -88,6 +88,40 @@ fn set_next_entry(id: &str) -> Result<()> {
     bail!("neither grub2-editenv nor grub-editenv found in PATH")
 }
 
+/// Return the GRUB menuentry index (0-based, newest-first) of the BLS entry
+/// whose file stem equals `target_id`.
+///
+/// Ubuntu's GRUB does not ship blscfg.mod, so cbootc generates a traditional
+/// grub.cfg via write_grub_menuentry_cfg, sorted newest-first by mtime.
+/// Ubuntu's GRUB does not reliably match `set default=<128-char-id>` against
+/// `menuentry --id <id>`, so we write the numeric index instead.
+fn grub_numeric_index(target_id: &str) -> Result<usize> {
+    let dir = Path::new(ENTRIES_DIR);
+    let mut all: Vec<(SystemTime, String)> = Vec::new();
+    for item in fs::read_dir(dir).with_context(|| format!("reading {ENTRIES_DIR}"))? {
+        let item = item.with_context(|| format!("iterating {ENTRIES_DIR}"))?;
+        let path = item.path();
+        if path.extension().and_then(|e| e.to_str()) != Some("conf") {
+            continue;
+        }
+        let mtime = item
+            .metadata()
+            .and_then(|m| m.modified())
+            .unwrap_or(SystemTime::UNIX_EPOCH);
+        let stem = path
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .unwrap_or("")
+            .to_owned();
+        all.push((mtime, stem));
+    }
+    // Newest-first — same order write_grub_menuentry_cfg uses (index 0 = default).
+    all.sort_by_key(|(t, _)| std::cmp::Reverse(*t));
+    all.iter()
+        .position(|(_, stem)| stem == target_id)
+        .with_context(|| format!("BLS entry {target_id} not found in {ENTRIES_DIR}"))
+}
+
 fn load_uki_entries() -> Result<Vec<BLSEntry>> {
     let dir = Path::new(EFI_LINUX_DIR);
     let mut entries = Vec::new();
@@ -118,17 +152,6 @@ fn load_uki_entries() -> Result<Vec<BLSEntry>> {
     Ok(entries)
 }
 
-fn set_next_entry_bootctl(id: &str) -> Result<()> {
-    let status = Command::new("bootctl")
-        .args(["set-next", id])
-        .status()
-        .context("spawning bootctl set-next")?;
-    if !status.success() {
-        bail!("bootctl set-next failed: {status}");
-    }
-    Ok(())
-}
-
 fn use_systemd_boot() -> bool {
     Path::new(EFI_LINUX_DIR).exists() && grubenv_path().is_none()
 }
@@ -156,9 +179,19 @@ pub fn run() -> Result<()> {
 
     let id = entry_id(&previous.path);
     if systemd_boot {
-        set_next_entry_bootctl(id)?;
-    } else {
+        crate::upgrade::set_loader_conf_default(std::path::Path::new("/boot/efi"), id)?;
+        crate::upgrade::bootctl_set_default(id)?;
+    } else if crate::install::has_grub2() {
+        // Fedora/RHEL: grub2's blscfg.mod reads BLS entries natively and
+        // matches next_entry=<digest> against each entry's --id.
         set_next_entry(id)?;
+    } else {
+        // Ubuntu/Debian: no blscfg.mod; grub.cfg is generated by
+        // write_grub_menuentry_cfg, sorted newest-first.  GRUB does not
+        // reliably match set default=<128-char-digest> against --id, so
+        // write the numeric position instead.
+        let index = grub_numeric_index(id)?;
+        set_next_entry(&index.to_string())?;
     }
 
     println!(

src/upgrade.rs

@@ -1,10 +1,56 @@
 use anyhow::{Context, Result};
 use chrono::Utc;
 use serde::{Deserialize, Serialize};
-use std::{fs, os::unix::fs::symlink, path::Path, path::PathBuf, process::Command};
+use std::{fs, io, os::unix::fs::symlink, path::Path, path::PathBuf, process::Command};
 
 use crate::{cfsctl, config, signing};
 
+/// Set the systemd-boot default via `bootctl set-default`, writing the
+/// `LoaderEntryDefault` EFI variable (which takes priority over `loader.conf`).
+/// Silently skips if `bootctl` is not found (container/install contexts without
+/// a writable efivarfs).
+pub(crate) fn bootctl_set_default(entry_id: &str) -> Result<()> {
+    let id_with_efi = format!("{entry_id}.efi");
+    match Command::new("bootctl")
+        .args(["set-default", &id_with_efi])
+        .status()
+    {
+        Ok(s) if s.success() => Ok(()),
+        Ok(s) => anyhow::bail!("bootctl set-default: exited {s}"),
+        Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
+        Err(e) => Err(e).context("spawning bootctl"),
+    }
+}
+
+/// Write (or update) the `default <entry_id>` line in
+/// `<esp>/loader/loader.conf`.  Used as a fallback for contexts without a
+/// writable efivarfs (container image builds, install from live media).
+pub(crate) fn set_loader_conf_default(esp: &Path, entry_id: &str) -> Result<()> {
+    let conf = esp.join("loader/loader.conf");
+    fs::create_dir_all(conf.parent().unwrap()).context("creating loader dir")?;
+    let existing = if conf.exists() {
+        fs::read_to_string(&conf).with_context(|| format!("reading {}", conf.display()))?
+    } else {
+        String::new()
+    };
+    let mut replaced = false;
+    let mut lines: Vec<String> = existing
+        .lines()
+        .map(|l| {
+            if l.starts_with("default ") || l == "default" {
+                replaced = true;
+                format!("default {entry_id}")
+            } else {
+                l.to_owned()
+            }
+        })
+        .collect();
+    if !replaced {
+        lines.push(format!("default {entry_id}"));
+    }
+    fs::write(&conf, lines.join("\n") + "\n").with_context(|| format!("writing {}", conf.display()))
+}
+
 const EFI_ESP: &str = "/boot/efi";
 // UKIs live on the ESP, not XBOOTLDR — systemd-boot always scans its own partition.
 const EFI_LINUX_DIR: &str = "/boot/efi/EFI/Linux";
@@ -65,12 +111,22 @@ pub fn run(reboot: bool) -> Result<()> {
             println!("Signing UKI ...");
             crate::install::sign_efi(&uki_path, Path::new(&sb.key), Path::new(&sb.cert))?;
         }
+        // Make the new UKI the permanent default.  Write both the EFI variable
+        // (via bootctl, highest priority) and loader.conf (fallback for contexts
+        // without a writable efivarfs, e.g. container image builds).
+        set_loader_conf_default(Path::new(EFI_ESP), &digest)?;
+        bootctl_set_default(&digest)?;
     } else {
         patch_bls_entry(Path::new(BOOT_DIR), &digest, &image_ref)?;
         if !crate::install::has_grub2() {
             // Ubuntu: regenerate menuentry-based grub.cfg so the new deployment
             // appears in the menu (blscfg.mod is not available on Ubuntu).
             write_grub_menuentry_cfg(Path::new(BOOT_DIR), crate::install::grub_dir())?;
+        } else {
+            // Fedora/RHEL: blscfg.mod reads BLS entries but selects the default
+            // based on grubenv `default`.  Without this update, GRUB continues
+            // to boot the previous entry regardless of the new BLS conf.
+            set_grub_default(&digest)?;
         }
     }
 
@@ -190,8 +246,9 @@ pub fn patch_bls_entry(bootdir: &Path, digest: &str, image_ref: &str) -> Result<
 /// `bootdir/loader/entries/`.  Used on distros (Ubuntu/Debian) that do not
 /// ship `blscfg.mod` in their GRUB package.
 ///
-/// Each entry gets `--id <digest>` so that rollback's `next_entry=<digest>`
-/// in grubenv correctly selects the previous deployment.
+/// Entries are sorted newest-first; index 0 is the default boot entry.
+/// rollback.rs writes `next_entry=<numeric-index>` (not the digest) for
+/// Ubuntu because GRUB does not reliably match long --id values.
 pub fn write_grub_menuentry_cfg(bootdir: &Path, grub_subdir: &str) -> Result<()> {
     let entries_dir = bootdir.join("loader/entries");
     let mut bls: Vec<(std::time::SystemTime, String, String, String)> = Vec::new();
@@ -333,6 +390,30 @@ fn write_state(digest: &str, manifest_digest: Option<&str>) -> Result<()> {
     fs::write(STATE_PATH, json).with_context(|| format!("writing {STATE_PATH}"))
 }
 
+/// Set the permanent GRUB default to `entry_id` via grub2-editenv/grub-editenv.
+/// If grubenv does not exist yet, skips silently (blscfg will fall back to its
+/// own sort order).
+fn set_grub_default(entry_id: &str) -> Result<()> {
+    let Some(grubenv) = ["/boot/grub2/grubenv", "/boot/grub/grubenv"]
+        .iter()
+        .find(|p| Path::new(p).exists())
+    else {
+        return Ok(());
+    };
+    for cmd in &["grub2-editenv", "grub-editenv"] {
+        match Command::new(cmd)
+            .args([*grubenv, "set", &format!("default={entry_id}")])
+            .status()
+        {
+            Ok(s) if s.success() => return Ok(()),
+            Ok(s) => anyhow::bail!("{cmd}: exited {s}"),
+            Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
+            Err(e) => return Err(e).with_context(|| format!("spawning {cmd}")),
+        }
+    }
+    anyhow::bail!("neither grub2-editenv nor grub-editenv found in PATH")
+}
+
 fn trigger_reboot() -> Result<()> {
     let status = Command::new("systemctl")
         .arg("reboot")