Added --virtiofsd command line option to the libvirt run command

fix: use overlayfs for ephemeral /var to handle large images with xattrs on virtiofs
Signed-off-by: Nick Downs <nickryand@gmail.com>

Author: nickryand

crates/integration-tests/src/tests/run_ephemeral.rs

@@ -406,7 +406,7 @@ integration_test!(test_run_ephemeral_virtiofs_mmap);
 /// Test that ephemeral VMs have the expected mount layout:
 /// - / is read-only virtiofs
 /// - /etc is overlayfs with tmpfs upper (writable)
-/// - /var is tmpfs (not overlayfs, so podman can use overlayfs inside)
+/// - /var is overlayfs with tmpfs upper (writable, instant setup for large /var trees)
 fn test_run_ephemeral_mount_layout() -> TestResult {
     let sh = shell()?;
     let bck = get_bck_command()?;
@@ -446,16 +446,17 @@ fn test_run_ephemeral_mount_layout() -> TestResult {
         etc_fstype
     );
 
-    // Check /var mount - should be tmpfs, NOT overlay
+    // Check /var mount - should be overlay (using overlayfs for instant setup,
+    // avoiding slow/failing cp -a of large /var trees on virtiofs)
     let var_fstype = cmd!(
         sh,
         "{bck} ephemeral run --rm --label {label} --execute 'findmnt -n -o FSTYPE /var' {image}"
     )
     .read()?;
     assert_eq!(
         var_fstype.trim(),
-        "tmpfs",
-        "/var should be tmpfs (not overlay), got: {}",
+        "overlay",
+        "/var should be overlay, got: {}",
         var_fstype
     );
 

crates/kit/src/cpio.rs

@@ -94,7 +94,7 @@ pub fn create_initramfs_units_cpio() -> io::Result<Vec<u8>> {
         ),
         File(
             "usr/lib/systemd/system/initrd-fs.target.d/bcvk-var-ephemeral.conf",
-            b"[Unit]\nWants=bcvk-var-ephemeral.service\n",
+            b"[Unit]\nRequires=bcvk-var-ephemeral.service\n",
         ),
         File(
             "usr/lib/systemd/system/initrd-fs.target.d/bcvk-copy-units.conf",

crates/kit/src/libvirt/base_disks.rs

@@ -18,6 +18,7 @@ pub fn find_or_create_base_disk(
     image_digest: &str,
     install_options: &InstallOptions,
     connect_uri: Option<&str>,
+    virtiofsd_binary: Option<String>,
 ) -> Result<Utf8PathBuf> {
     let metadata = DiskImageMetadata::from(install_options, image_digest, source_image);
     let cache_hash = metadata.compute_cache_hash();
@@ -67,6 +68,7 @@ pub fn find_or_create_base_disk(
         image_digest,
         install_options,
         connect_uri,
+        virtiofsd_binary,
     )?;
 
     Ok(base_disk_path)
@@ -79,6 +81,7 @@ fn create_base_disk(
     image_digest: &str,
     install_options: &InstallOptions,
     connect_uri: Option<&str>,
+    virtiofsd_binary: Option<String>,
 ) -> Result<()> {
     use crate::run_ephemeral::CommonVmOpts;
     use crate::to_disk::{Format, ToDiskAdditionalOpts, ToDiskOpts};
@@ -116,6 +119,7 @@ fn create_base_disk(
                 memory: crate::common_opts::MemoryOpts {
                     memory: super::LIBVIRT_DEFAULT_MEMORY.to_string(),
                 },
+                virtiofsd_binary,
                 ..Default::default()
             },
             ..Default::default()

crates/kit/src/libvirt/base_disks_cli.rs

@@ -77,6 +77,7 @@ pub fn run_create(
         &image_digest,
         &opts.install_options,
         connect_uri,
+        None,
     )?;
     println!("Created base disk: {path}");
 

crates/kit/src/libvirt/run.rs

@@ -309,6 +309,10 @@ pub struct LibvirtRunOpts {
     #[clap(long = "ignition")]
     pub ignition_config: Option<Utf8PathBuf>,
 
+    /// Path to virtiofsd binary (overrides auto-detection for disk creation)
+    #[clap(long = "virtiofsd", env = "VIRTIOFSD_BIN")]
+    pub virtiofsd_binary: Option<String>,
+
     /// Log virtio console (OS/journald on hvc0) to this file (created if absent)
     #[clap(long = "console-log")]
     pub console_log: Option<Utf8PathBuf>,
@@ -498,6 +502,7 @@ pub fn run(global_opts: &crate::libvirt::LibvirtOptions, mut opts: LibvirtRunOpt
         &image_digest,
         &opts.install,
         connect_uri,
+        opts.virtiofsd_binary.take(),
     )
     .with_context(|| "Failed to find or create base disk")?;
 

crates/kit/src/units/bcvk-var-ephemeral.service

@@ -10,6 +10,10 @@ Requires=sysroot.mount
 Type=oneshot
 RemainAfterExit=yes
 TimeoutStartSec=60
-ExecStart=/usr/bin/mkdir -p /run/var-ephemeral
-ExecStart=/usr/bin/cp -a /sysroot/var/. /run/var-ephemeral/
-ExecStart=/usr/bin/mount --bind /run/var-ephemeral /sysroot/var
+# Use overlayfs like /etc to avoid copying large files
+# from virtiofs to tmpfs. The cp -a approach fails on virtiofs due to xattr issues
+# and is prohibitively slow for large /var trees.
+# Bind-mount /sysroot/var to a separate location first (same technique as /etc overlay).
+ExecStart=/usr/bin/mkdir -p /run/var-lower /run/var-upper /run/var-work
+ExecStart=/usr/bin/mount --bind /sysroot/var /run/var-lower
+ExecStart=/usr/bin/mount -t overlay overlay -o lowerdir=/run/var-lower,upperdir=/run/var-upper,workdir=/run/var-work,index=off,metacopy=off /sysroot/var