xtask: Add validate-composefs-digest for debugging mtime issues

Add a command to build a sealed image and compare composefs dumpfiles
between the build-time view (mounted filesystem) and install-time view
(OCI tar layer processing). This helps debug mtime and metadata
discrepancies that cause sealed boot failures.

Usage:
  cargo xtask validate-composefs-digest
  cargo xtask validate-composefs-digest --no-cache  # force clean build

Output files are written to target/validate-digest/:
  - build.dumpfile: from mounted filesystem (what seal-uki sees)
  - storage.dumpfile: from containers-storage (what bootc upgrade sees)
  - diff.txt: differences if any

Signed-off-by: Colin Walters <walters@verbum.org>

Assisted-by: OpenCode (Opus 4.5)
Signed-off-by: Colin Walters <walters@verbum.org>

Author: cgwalters

crates/xtask/src/xtask.rs

@@ -54,6 +54,20 @@ enum Commands {
     TmtProvision(TmtProvisionArgs),
     /// Check build system properties (e.g., reproducible builds)
     CheckBuildsys,
+    /// Validate composefs digests match between build-time and install-time views
+    ValidateComposefsDigest(ValidateComposefsDigestArgs),
+}
+
+/// Arguments for validate-composefs-digest command
+#[derive(Debug, Args)]
+pub(crate) struct ValidateComposefsDigestArgs {
+    /// Force a clean build (no podman cache)
+    #[arg(long)]
+    pub(crate) no_cache: bool,
+
+    /// Base image to use
+    #[arg(long, default_value = "quay.io/centos-bootc/centos-bootc:stream10")]
+    pub(crate) base: String,
 }
 
 /// Arguments for run-tmt command
@@ -139,6 +153,7 @@ fn try_main() -> Result<()> {
         Commands::RunTmt(args) => tmt::run_tmt(&sh, &args),
         Commands::TmtProvision(args) => tmt::tmt_provision(&sh, &args),
         Commands::CheckBuildsys => buildsys::check_buildsys(&sh, "Dockerfile".into()),
+        Commands::ValidateComposefsDigest(args) => validate_composefs_digest(&sh, &args),
     }
 }
 
@@ -406,3 +421,110 @@ fn update_generated(sh: &Shell) -> Result<()> {
 
     Ok(())
 }
+
+/// Validate that composefs digests match between build-time and install-time views.
+///
+/// This builds a sealed image and compares dumpfiles generated from:
+/// 1. The mounted filesystem (what seal-uki sees at build time)
+/// 2. The OCI tar layers in containers-storage (what bootc upgrade sees)
+///
+/// This helps debug mtime and metadata discrepancies that cause sealed boot failures.
+#[context("Validating composefs digest")]
+fn validate_composefs_digest(sh: &Shell, args: &ValidateComposefsDigestArgs) -> Result<()> {
+    let variant = "composefs-sealeduki-sdboot";
+    let out_dir = Utf8Path::new("target/validate-digest");
+
+    // Ensure packages are built
+    if !Utf8Path::new("target/packages").exists() {
+        anyhow::bail!("target/packages not found. Run 'just package' first.");
+    }
+
+    // Ensure secureboot keys exist
+    if !Utf8Path::new("target/test-secureboot/db.key").exists() {
+        cmd!(sh, "./hack/generate-secureboot-keys").run()?;
+    }
+
+    std::fs::create_dir_all(out_dir)?;
+    let pkg_path = std::fs::canonicalize("target/packages")?;
+    let out_dir_abs = std::fs::canonicalize(out_dir)?;
+
+    // Build the base-penultimate stage
+    println!("=== Building base-penultimate stage ===");
+    let no_cache = args.no_cache.then_some("--no-cache");
+    let base = &args.base;
+    let pkg_vol = format!("{}:/run/packages:ro,z", pkg_path.display());
+    cmd!(
+        sh,
+        "podman build {no_cache...}
+            -v {pkg_vol}
+            --build-arg=base={base}
+            --build-arg=variant={variant}
+            --cap-add=all
+            --security-opt=label=type:container_runtime_t
+            --device=/dev/fuse
+            --secret=id=secureboot_key,src=target/test-secureboot/db.key
+            --secret=id=secureboot_cert,src=target/test-secureboot/db.crt
+            --target=base-penultimate
+            -t localhost/bootc-penultimate
+            ."
+    )
+    .run()?;
+
+    // Generate dumpfile from mounted filesystem (build-time view)
+    println!("=== Generating build-time dumpfile (from mounted filesystem) ===");
+    let build_dumpfile = out_dir_abs.join("build.dumpfile");
+    let out_vol = format!("{}:/out:z", out_dir_abs.display());
+    cmd!(
+        sh,
+        "podman run --rm --privileged
+            -v {out_vol}
+            --mount=type=image,source=localhost/bootc-penultimate,target=/target
+            localhost/bootc-penultimate
+            bootc container compute-composefs-digest /target
+            --write-dumpfile-to /out/build.dumpfile"
+    )
+    .run()?;
+    println!("Build-time dumpfile: {}", build_dumpfile.display());
+
+    // Generate dumpfile from containers-storage (install-time view)
+    println!("=== Generating install-time dumpfile (from containers-storage) ===");
+    let format_arg = "{{.Store.GraphRoot}}";
+    let graphroot = cmd!(sh, "podman system info -f {format_arg}").read()?;
+    let graphroot = graphroot.trim();
+    let storage_dumpfile = out_dir_abs.join("storage.dumpfile");
+    let storage_vol = format!("{graphroot}:/run/host-container-storage:ro");
+    cmd!(
+        sh,
+        "podman run --rm --privileged --security-opt=label=disable
+            -v {out_vol}
+            -v {storage_vol}
+            -v /sys:/sys:ro
+            --tmpfs=/var
+            localhost/bootc-penultimate
+            bootc container compute-composefs-digest-from-storage
+            --write-dumpfile-to /out/storage.dumpfile"
+    )
+    .run()?;
+    println!("Install-time dumpfile: {}", storage_dumpfile.display());
+
+    // Compare dumpfiles
+    println!("=== Comparing dumpfiles ===");
+    let diff_result = cmd!(sh, "diff -u {build_dumpfile} {storage_dumpfile}")
+        .ignore_status()
+        .output()?;
+
+    if diff_result.status.success() {
+        println!("SUCCESS: Dumpfiles match!");
+        Ok(())
+    } else {
+        let diff_file = out_dir_abs.join("diff.txt");
+        std::fs::write(&diff_file, &diff_result.stdout)?;
+        println!("MISMATCH: Dumpfiles differ! See {}", diff_file.display());
+        println!("First 50 lines of diff:");
+        let stdout = String::from_utf8_lossy(&diff_result.stdout);
+        for line in stdout.lines().take(50) {
+            println!("{line}");
+        }
+        anyhow::bail!("Composefs digest mismatch");
+    }
+}