unified-storage: Implement three-store pipeline for ostree backend

The goal of unified storage is to make the same on-disk layer data
simultaneously visible to containers-storage (for \`podman run\`),
composefs (for the boot overlay), and ostree (for deployment tracking),
using reflinks so layers are stored once regardless of how many stores
reference them.

This wires the full pipeline into the ostree backend. When a system has
unified storage enabled — either at install time via the
[install.storage] config key, or post-install via \`bootc image
set-unified\` — upgrades and switches route through pull_via_composefs:

  Stage 1: pull image into bootc-owned containers-storage
  Stage 2: zero-copy reflink import into the composefs OCI repo
  Stage 3: synthesize an ostree commit from the composefs tree

Whether unified storage is active is tracked by composefs/bootc.json
(BootcRepoMeta). This replaces the previous heuristic that checked
per-image presence in containers-storage, which broke when switching to
a new image reference.

The install config gains a storage.unified key with three values:
disabled (default), enabled (fail if reflinks unavailable), and
enabled-with-copy (copy fallback). This lets an image opt into unified
storage without requiring a CLI flag to be threaded through every
installer.

bootc image list cross-references composefs tags against containers-storage
by config digest to report images as unified (in all three stores) or
partial (cstorage only, composefs import pending). bootc internals fsck
images checks consistency and --repair restores cstorage from composefs
when needed. The cstorage GC is extended to protect images that have
composefs tags, since the composefs splitstreams reference the cstorage
layer data — pruning one without the other would corrupt the repo.

Assisted-by: OpenCode (claude-sonnet-4-6@default)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

.github/workflows/ci.yml

@@ -158,6 +158,13 @@ jobs:
           # These tests may mutate the system live so we can't run in parallel
           sudo bootc-integration-tests system-reinstall localhost/bootc --test-threads=1
 
+          # Unified storage case
+          sudo podman build -t localhost/bootc-unified-storage -f ci/Containerfile.install-unified-storage
+          sudo podman run --privileged --pid=host localhost/bootc-unified-storage bootc install to-existing-root --stateroot=unified-storage --acknowledge-destructive --skip-fetch-check
+          # Verify unified storage was activated; composefs/bootc.json is written relative to
+          # the target physical root (/target bind-mounted to host /), so the file appears at /composefs/bootc.json
+          sudo test -f /composefs/bootc.json
+
           # And the fsverity case
           sudo podman run --privileged --pid=host localhost/bootc-fsverity bootc install to-existing-root --stateroot=other \
             --acknowledge-destructive --skip-fetch-check

Justfile

@@ -290,12 +290,27 @@ test-tmt-baseconfig baseconfig *ARGS:
         --seal-state={{seal_state}} \
         {{base_img}} readonly {{ARGS}}
 
+# Run unified-storage baseconfig test (works with ostree variant, no composefs required)
+[group('testing')]
+test-tmt-baseconfig-unified-storage *ARGS:
+    just baseconfigs=unified-storage build
+    just baseconfigs=unified-storage _build-upgrade-image
+    cargo xtask run-tmt \
+        --env=BOOTC_baseconfigs=unified-storage \
+        --upgrade-image={{upgrade_img}} \
+        --bootloader={{bootloader}} \
+        --filesystem={{filesystem}} \
+        --boot-type={{boot_type}} \
+        --seal-state={{seal_state}} \
+        {{base_img}} readonly {{ARGS}}
+
 # Run readonly tests for all standard baseconfigs
 [group('testing')]
 test-baseconfigs *ARGS:
     just test-tmt-baseconfig etc-transient {{ARGS}}
     just test-tmt-baseconfig root-transient {{ARGS}}
     just test-tmt-baseconfig var-volatile {{ARGS}}
+    just test-tmt-baseconfig-unified-storage {{ARGS}}
 
 # Run tmt tests on Fedora CoreOS
 [group('testing')]

ci/Containerfile.install-unified-storage

@@ -0,0 +1,8 @@
+# Enable unified storage (composefs+ostree) at install time via image-embedded config
+FROM localhost/bootc-install
+RUN <<EORUN
+set -xeuo pipefail
+mkdir -p /usr/lib/bootc/install
+printf '[install.storage]\nunified = "enabled-with-copies"\n' > /usr/lib/bootc/install/00-storage.toml
+bootc container lint
+EORUN

contrib/packaging/inject-baseconfig

@@ -11,21 +11,8 @@ if [ -z "${BASECONFIGS}" ]; then
   exit 0
 fi
 
-# setup-root-conf.toml is composefs-specific; ostree uses prepare-root.conf
-# which has a different (INI) format and different option names.
-case "${VARIANT}" in
-  composefs*)
-    TARGET="/usr/lib/composefs/setup-root-conf.toml"
-    ;;
-  *)
-    echo "inject-baseconfig: baseconfigs not supported for variant '${VARIANT}'" >&2
-    exit 1
-    ;;
-esac
-
-mkdir -p "$(dirname "${TARGET}")"
-
-# Split on commas and process each token
+# Split and process tokens; unified-storage is handled before the variant check
+# because it is backend-independent (works with both ostree and composefs).
 IFS=',' read -ra TOKENS <<< "${BASECONFIGS}"
 for raw_token in "${TOKENS[@]}"; do
   # Trim leading/trailing spaces
@@ -35,27 +22,50 @@ for raw_token in "${TOKENS[@]}"; do
   [ -z "${token}" ] && continue
 
   case "${token}" in
-    etc-transient)
-      printf '[etc]\ntransient = true\n' >> "${TARGET}"
-      ;;
-    root-transient)
-      printf '[root]\ntransient = true\n' >> "${TARGET}"
-      ;;
-    var-volatile)
-      # Mount /var as a fresh tmpfs on every boot via systemd.volatile=state.
-      # bootc-root-setup detects this karg in the initramfs and automatically
-      # skips the /var state bind-mount, leaving /var as an empty directory
-      # from the composefs image.  systemd-fstab-generator then mounts a fresh
-      # tmpfs there at local-fs.target.  Using a plain tmpfs avoids the
-      # overlayfs-on-overlayfs restriction that breaks tools like podman which
-      # use overlayfs under /var/lib/containers.
-      mkdir -p /usr/lib/bootc/kargs.d
-      printf 'kargs = ["systemd.volatile=state"]\n' \
-        > /usr/lib/bootc/kargs.d/50-var-volatile.toml
+    unified-storage)
+      # Write the bootc install config to enable unified storage at install time.
+      # This is backend-independent and works with both ostree and composefs variants.
+      mkdir -p /usr/lib/bootc/install
+      printf '[install.storage]\nunified = "enabled-with-copy"\n' \
+        > /usr/lib/bootc/install/00-storage.toml
       ;;
     *)
-      echo "Unknown baseconfig: ${token}" >&2
-      exit 1
+      # All other tokens require the composefs variant (they write to composefs-specific paths).
+      # Validate variant here, not at the top, so unified-storage can run on any variant.
+      case "${VARIANT}" in
+        composefs*)
+          TARGET="/usr/lib/composefs/setup-root-conf.toml"
+          mkdir -p "$(dirname "${TARGET}")"
+          ;;
+        *)
+          echo "inject-baseconfig: baseconfig '${token}' not supported for variant '${VARIANT}'" >&2
+          exit 1
+          ;;
+      esac
+      case "${token}" in
+        etc-transient)
+          printf '[etc]\ntransient = true\n' >> "${TARGET}"
+          ;;
+        root-transient)
+          printf '[root]\ntransient = true\n' >> "${TARGET}"
+          ;;
+        var-volatile)
+          # Mount /var as a fresh tmpfs on every boot via systemd.volatile=state.
+          # bootc-root-setup detects this karg in the initramfs and automatically
+          # skips the /var state bind-mount, leaving /var as an empty directory
+          # from the composefs image.  systemd-fstab-generator then mounts a fresh
+          # tmpfs there at local-fs.target.  Using a plain tmpfs avoids the
+          # overlayfs-on-overlayfs restriction that breaks tools like podman which
+          # use overlayfs under /var/lib/containers.
+          mkdir -p /usr/lib/bootc/kargs.d
+          printf 'kargs = ["systemd.volatile=state"]\n' \
+            > /usr/lib/bootc/kargs.d/50-var-volatile.toml
+          ;;
+        *)
+          echo "Unknown baseconfig: ${token}" >&2
+          exit 1
+          ;;
+      esac
       ;;
   esac
 done

crates/lib/src/bootc_composefs/export.rs

@@ -7,44 +7,27 @@ use composefs_ctl::composefs;
 use composefs_ctl::composefs_oci;
 use composefs_oci::open_config;
 use ocidir::{OciDir, oci_spec::image::Platform};
+use ostree_ext::container::ImageReference;
 use ostree_ext::container::Transport;
 use ostree_ext::container::skopeo;
 use tar::EntryType;
 
 use crate::image::get_imgrefs_for_copy;
 use crate::{
-    bootc_composefs::status::{get_composefs_status, get_imginfo},
-    store::{BootedComposefs, Storage},
+    bootc_composefs::status::{ImgConfigManifest, get_composefs_status, get_imginfo},
+    store::{BootedComposefs, ComposefsRepository, Storage},
 };
 
-/// Exports a composefs repository to a container image in containers-storage:
-pub async fn export_repo_to_image(
-    storage: &Storage,
-    booted_cfs: &BootedComposefs,
-    source: Option<&str>,
-    target: Option<&str>,
+/// Streams a composefs OCI image out to a destination image reference.
+///
+/// Given a composefs repository handle and image metadata (manifest + config),
+/// reconstructs the container image by reading layer data from the composefs
+/// splitstreams and copies the assembled OCI image to `dest_imgref` via skopeo.
+pub(crate) async fn export_composefs_to_dest(
+    composefs_repo: &ComposefsRepository,
+    imginfo: &ImgConfigManifest,
+    dest_imgref: &ImageReference,
 ) -> Result<()> {
-    let host = get_composefs_status(storage, booted_cfs).await?;
-
-    let (source, dest_imgref) = get_imgrefs_for_copy(&host, source, target).await?;
-
-    let mut depl_verity = None;
-
-    for depl in host.list_deployments() {
-        let img = &depl.image.as_ref().unwrap().image;
-
-        // Not checking transport here as we'll be pulling from the repo anyway
-        // So, image name is all we need
-        if img.image == source.name {
-            depl_verity = Some(depl.require_composefs()?.verity.clone());
-            break;
-        }
-    }
-
-    let depl_verity = depl_verity.ok_or_else(|| anyhow::anyhow!("Image {source} not found"))?;
-
-    let imginfo = get_imginfo(storage, &depl_verity)?;
-
     let config_digest = imginfo.manifest.config().digest().clone();
 
     let var_tmp =
@@ -54,7 +37,7 @@ pub async fn export_repo_to_image(
     let oci_dir = OciDir::ensure(tmpdir.try_clone()?).context("Opening OCI")?;
 
     // Use composefs_oci::open_config to get the config and layer map
-    let open = open_config(&*booted_cfs.repo, &config_digest, None).context("Opening config")?;
+    let open = open_config(composefs_repo, &config_digest, None).context("Opening config")?;
     let config = open.config;
     let layer_map = open.layer_refs;
 
@@ -77,7 +60,7 @@ pub async fn export_repo_to_image(
             .get(old_diff_id.as_str())
             .ok_or_else(|| anyhow::anyhow!("Layer {old_diff_id} not found in config"))?;
 
-        let mut layer_stream = booted_cfs.repo.open_stream("", Some(layer_verity), None)?;
+        let mut layer_stream = composefs_repo.open_stream("", Some(layer_verity), None)?;
 
         let mut layer_writer = oci_dir.create_layer(None)?;
         layer_writer.follow_symlinks(false);
@@ -113,7 +96,7 @@ pub async fn export_repo_to_image(
             match layer_stream.read_exact(size as usize, ((size as usize) + 511) & !511)? {
                 SplitStreamData::External(obj_id) => match header.entry_type() {
                     EntryType::Regular | EntryType::Continuous => {
-                        let file = File::from(booted_cfs.repo.open_object(&obj_id)?);
+                        let file = File::from(composefs_repo.open_object(&obj_id)?);
 
                         layer_writer
                             .append(&header, file)
@@ -196,7 +179,7 @@ pub async fn export_repo_to_image(
 
     skopeo::copy(
         &tempoci,
-        &dest_imgref,
+        dest_imgref,
         None,
         Some((
             std::sync::Arc::new(tmpdir.try_clone()?.into()),
@@ -208,3 +191,37 @@ pub async fn export_repo_to_image(
 
     Ok(())
 }
+
+/// Exports a composefs repository to a container image in containers-storage:
+pub async fn export_repo_to_image(
+    storage: &Storage,
+    booted_cfs: &BootedComposefs,
+    source: Option<&str>,
+    target: Option<&str>,
+) -> Result<()> {
+    let host = get_composefs_status(storage, booted_cfs).await?;
+
+    let (source, dest_imgref) = get_imgrefs_for_copy(&host, source, target).await?;
+
+    let mut depl_verity = None;
+
+    for depl in host.list_deployments() {
+        let img = &depl.image.as_ref().unwrap().image;
+
+        // Not checking transport here as we'll be pulling from the repo anyway
+        // So, image name is all we need
+        if img.image == source.name {
+            depl_verity = Some(depl.require_composefs()?.verity.clone());
+            break;
+        }
+    }
+
+    let depl_verity = depl_verity.ok_or_else(|| anyhow::anyhow!("Image {source} not found"))?;
+
+    let imginfo = get_imginfo(storage, &depl_verity)?;
+
+    println!("Copying local image {source} to {dest_imgref} ...");
+    export_composefs_to_dest(&booted_cfs.repo, &imginfo, &dest_imgref).await?;
+    println!("Pushed: {dest_imgref}");
+    Ok(())
+}

crates/lib/src/bootc_composefs/repo.rs

@@ -41,6 +41,7 @@ pub(crate) async fn initialize_composefs_repository(
     root_setup: &RootSetup,
     allow_missing_fsverity: bool,
     use_unified: bool,
+    local_fetch: LocalFetchOpt,
 ) -> Result<PullResult<Sha512HashValue>> {
     const COMPOSEFS_REPO_INIT_JOURNAL_ID: &str = "5d4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9";
 
@@ -99,7 +100,7 @@ pub(crate) async fn initialize_composefs_repository(
         let imgstore = CStorage::create(rootfs_dir, &run, sepolicy.as_ref())?;
         let storage_path = root_setup.physical_root_path.join(CStorage::subpath());
 
-        let r = pull_composefs_unified(&imgstore, storage_path.as_str(), &repo, &imgref).await?;
+        let r = pull_composefs_unified(&imgstore, storage_path.as_str(), &repo, &imgref, local_fetch).await?;
 
         // SELinux-label the containers-storage now that all pulls are done.
         imgstore
@@ -179,16 +180,18 @@ async fn pull_composefs_unified(
     storage_path: &str,
     repo: &Arc<crate::store::ComposefsRepository>,
     imgref: &containers_image_proxy::ImageReference,
+    local_fetch: LocalFetchOpt,
 ) -> Result<PullResult<Sha512HashValue>> {
     let image = &imgref.name;
 
     // Stage 1: get the image into bootc-owned containers-storage.
     if imgref.transport == containers_image_proxy::Transport::ContainerStorage {
-        // The image is in the default containers-storage (/var/lib/containers/storage).
-        // Copy it into bootc-owned storage.
+        // The image is in a containers-storage instance — either the default
+        // /var/lib/containers/storage or an additional image store advertised
+        // via STORAGE_OPTS (e.g. the bcvk virtiofs mount).
         tracing::info!("Unified pull: copying {image} from host containers-storage");
         imgstore
-            .pull_from_host_storage(image)
+            .pull_from_containers_storage(image)
             .await
             .context("Copying image from host containers-storage into bootc storage")?;
     } else {
@@ -210,11 +213,11 @@ async fn pull_composefs_unified(
     let storage = std::path::Path::new(storage_path);
     let pull_opts = PullOptions {
         // The image is already in bootc-owned containers-storage at this point
-        // (placed there by Stage 1 of the unified pull). Use ZeroCopy so we
-        // actually import via reflink/hardlink and fail loudly if that isn't
-        // possible — a plain copy fallback here would mean Stage 1 and Stage 2
-        // are on different filesystems or the storage root is wrong.
-        local_fetch: LocalFetchOpt::ZeroCopy,
+        // (placed there by Stage 1 of the unified pull). CopyMode controls
+        // whether a fallback to byte copies is acceptable:
+        // ZeroCopy → fail if reflinks unavailable (storage.unified = "enabled")
+        // IfPossible → byte-copy fallback ok (storage.unified = "enabled-with-copy")
+        local_fetch,
         storage_root: Some(storage),
         ..Default::default()
     };
@@ -240,6 +243,7 @@ pub(crate) async fn pull_composefs_repo(
     spec_imgref: &crate::spec::ImageReference,
     allow_missing_fsverity: bool,
     use_unified: bool,
+    local_fetch: LocalFetchOpt,
 ) -> Result<PullRepoResult> {
     const COMPOSEFS_PULL_JOURNAL_ID: &str = "4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9f8";
 
@@ -290,7 +294,7 @@ pub(crate) async fn pull_composefs_repo(
         let imgstore = CStorage::create(&rootfs_dir, &run, sepolicy.as_ref())?;
         let storage_path = format!("/sysroot/{}", CStorage::subpath());
 
-        pull_composefs_unified(&imgstore, &storage_path, &repo, &imgref).await?
+        pull_composefs_unified(&imgstore, &storage_path, &repo, &imgref, local_fetch).await?
     } else {
         pull_composefs_direct(&repo, &imgref).await?
     };

crates/lib/src/bootc_composefs/status.rs

@@ -970,6 +970,12 @@ async fn composefs_deployment_status_from(
 
     host.status.usr_overlay = get_composefs_usr_overlay_status().ok().flatten();
 
+    // Populate storage status from bootc repo metadata stored on the physical root.
+    host.status.storage = crate::store::BootcRepoMeta::read(&storage.physical_root)
+        .ok()
+        .flatten()
+        .map(|meta| crate::spec::StorageStatus { unified: meta.unified });
+
     set_soft_reboot_capability(storage, &mut host, sorted_bls_config, cmdline)?;
 
     Ok(host)

crates/lib/src/bootc_composefs/switch.rs

@@ -51,23 +51,11 @@ pub(crate) async fn switch_composefs(
     let repo = &*booted_cfs.repo;
     let (image, img_config) = is_image_pulled(repo, &target_imgref).await?;
 
-    // Use unified storage if explicitly requested, or auto-detect: either the
-    // target image is already in bootc-owned containers-storage, OR the booted
-    // image is — which means the user has opted into unified storage and all
-    // subsequent operations (including switch to a new image) should use it.
-    let use_unified = if opts.unified_storage_exp {
-        true
-    } else {
-        let booted_imgref = host.spec.image.as_ref();
-        let booted_unified = if let Some(booted) = booted_imgref {
-            crate::deploy::image_exists_in_unified_storage(storage, booted).await?
-        } else {
-            false
-        };
-        let target_unified =
-            crate::deploy::image_exists_in_unified_storage(storage, &target_imgref).await?;
-        booted_unified || target_unified
-    };
+    // Use unified storage if explicitly requested via flag, or if the
+    // composefs/bootc.json marker says unified storage is enabled on this system.
+    let use_unified = opts.unified_storage_exp
+        || crate::deploy::unified_storage_enabled(storage)
+            .context("Checking unified storage flag")?;
 
     let do_upgrade_opts = DoUpgradeOpts {
         soft_reboot: opts.soft_reboot,