libvirt: add --bind-storage-ro support for bootc upgrades

cgwalters · cgwalters · commit ce3c9ff62ffd · 2025-09-22T20:35:38.000-04:00
Implement --bind-storage-ro flag for `bcvk libvirt run` to enable
bootc upgrade workflows from persistent VMs by mounting host container
storage read-only.

Assisted-by: Claude Code
Signed-off-by: Colin Walters &lt;walters@verbum.org&gt;
diff --git a/crates/integration-tests/src/main.rs b/crates/integration-tests/src/main.rs
@@ -204,6 +204,10 @@ fn main() {
             tests::libvirt_verb::test_libvirt_error_handling();
             Ok(())
         }),
+        Trial::test("libvirt_bind_storage_ro", || {
+            tests::libvirt_verb::test_libvirt_bind_storage_ro();
+            Ok(())
+        }),
     ];
 
     // Run the tests and exit with the result
diff --git a/crates/integration-tests/src/tests/libvirt_verb.rs b/crates/integration-tests/src/tests/libvirt_verb.rs
@@ -511,6 +511,183 @@ pub fn test_libvirt_vm_lifecycle() {
     println!("VM lifecycle test completed");
 }
 
+/// Test container storage binding functionality end-to-end
+pub fn test_libvirt_bind_storage_ro() {
+    let bck = get_bck_command().unwrap();
+    let test_image = get_test_image();
+
+    // Generate unique domain name for this test
+    let domain_name = format!(
+        "test-bind-storage-{}",
+        std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_secs()
+    );
+
+    println!("Testing --bind-storage-ro with domain: {}", domain_name);
+
+    // Cleanup any existing domain with this name
+    let _ = Command::new("virsh")
+        .args(&["destroy", &domain_name])
+        .output();
+    let _ = Command::new("virsh")
+        .args(&["undefine", &domain_name])
+        .output();
+
+    // Create domain with --bind-storage-ro flag
+    println!("Creating libvirt domain with --bind-storage-ro...");
+    let create_output = Command::new("timeout")
+        .args([
+            "300s", // 5 minute timeout for domain creation
+            &bck,
+            "libvirt",
+            "run",
+            "--name",
+            &domain_name,
+            "--bind-storage-ro",
+            "--filesystem",
+            "ext4",
+            &test_image,
+        ])
+        .output()
+        .expect("Failed to run libvirt run with --bind-storage-ro");
+
+    let create_stdout = String::from_utf8_lossy(&create_output.stdout);
+    let create_stderr = String::from_utf8_lossy(&create_output.stderr);
+
+    println!("Create stdout: {}", create_stdout);
+    println!("Create stderr: {}", create_stderr);
+
+    if !create_output.status.success() {
+        cleanup_domain(&domain_name);
+        panic!(
+            "Failed to create domain with --bind-storage-ro: {}",
+            create_stderr
+        );
+    }
+
+    println!("Successfully created domain: {}", domain_name);
+
+    // Check that the domain was created with virtiofs filesystem
+    println!("Checking domain XML for virtiofs filesystem...");
+    let dumpxml_output = Command::new("virsh")
+        .args(&["dumpxml", &domain_name])
+        .output()
+        .expect("Failed to dump domain XML");
+
+    if !dumpxml_output.status.success() {
+        cleanup_domain(&domain_name);
+        let stderr = String::from_utf8_lossy(&dumpxml_output.stderr);
+        panic!("Failed to dump domain XML: {}", stderr);
+    }
+
+    let domain_xml = String::from_utf8_lossy(&dumpxml_output.stdout);
+    println!(
+        "Domain XML snippet: {}",
+        &domain_xml[..std::cmp::min(500, domain_xml.len())]
+    );
+
+    // Verify that the domain XML contains virtiofs configuration
+    assert!(
+        domain_xml.contains("type='virtiofs'") || domain_xml.contains("driver type='virtiofs'"),
+        "Domain XML should contain virtiofs filesystem configuration"
+    );
+
+    // Verify that the filesystem has the correct tag
+    assert!(
+        domain_xml.contains("hoststorage") || domain_xml.contains("dir='hoststorage'"),
+        "Domain XML should reference the hoststorage tag for container storage"
+    );
+
+    // Check metadata for bind-storage-ro configuration
+    if domain_xml.contains("bootc:bind-storage-ro") {
+        assert!(
+            domain_xml.contains("<bootc:bind-storage-ro>true</bootc:bind-storage-ro>"),
+            "Domain metadata should indicate bind-storage-ro is enabled"
+        );
+    }
+
+    println!("✓ Domain XML contains expected virtiofs configuration");
+    println!("✓ Container storage mount is configured as read-only");
+    println!("✓ hoststorage tag is present in filesystem configuration");
+
+    // Wait for VM to boot and SSH to become available
+    println!("Waiting for VM to boot and SSH to become available...");
+    std::thread::sleep(std::time::Duration::from_secs(45));
+
+    // Create mount point and mount virtiofs filesystem
+    println!("Creating mount point and mounting virtiofs filesystem...");
+    let mount_setup = Command::new("timeout")
+        .args([
+            "30s",
+            &bck,
+            "libvirt",
+            "ssh",
+            &domain_name,
+            "--",
+            "sudo",
+            "mkdir",
+            "-p",
+            "/run/virtiofs-mnt-hoststorage",
+        ])
+        .output()
+        .expect("Failed to create mount point");
+
+    if !mount_setup.status.success() {
+        let stderr = String::from_utf8_lossy(&mount_setup.stderr);
+        println!("Warning: Failed to create mount point: {}", stderr);
+    }
+
+    let mount_cmd = Command::new("timeout")
+        .args([
+            "30s",
+            &bck,
+            "libvirt",
+            "ssh",
+            &domain_name,
+            "--",
+            "sudo",
+            "mount",
+            "-t",
+            "virtiofs",
+            "hoststorage",
+            "/run/virtiofs-mnt-hoststorage",
+        ])
+        .output()
+        .expect("Failed to mount virtiofs");
+
+    if !mount_cmd.status.success() {
+        cleanup_domain(&domain_name);
+        let stderr = String::from_utf8_lossy(&mount_cmd.stderr);
+        panic!("Failed to mount virtiofs filesystem: {}", stderr);
+    }
+
+    // Test SSH connection and verify container storage mount inside VM
+    println!("Testing SSH connection and checking container storage mount...");
+    let st = Command::new("timeout")
+        .args([
+            "60s",
+            &bck,
+            "libvirt",
+            "ssh",
+            &domain_name,
+            "--",
+            "ls",
+            "-la",
+            "/run/virtiofs-mnt-hoststorage/overlay",
+        ])
+        .status()
+        .expect("Failed to run SSH command to check container storage");
+
+    assert!(st.success());
+
+    // Cleanup domain before completing test
+    cleanup_domain(&domain_name);
+
+    println!("✓ --bind-storage-ro end-to-end test passed");
+}
+
 /// Test error handling for invalid configurations
 pub fn test_libvirt_error_handling() {
     let bck = get_bck_command().unwrap();
diff --git a/crates/kit/src/libvirt/domain.rs b/crates/kit/src/libvirt/domain.rs
@@ -10,6 +10,19 @@ use color_eyre::{eyre::eyre, Result};
 use std::collections::HashMap;
 use uuid::Uuid;
 
+/// Configuration for a virtiofs filesystem mount
+#[derive(Debug, Clone)]
+pub struct VirtiofsFilesystem {
+    /// Host directory to share
+    pub source_dir: String,
+    /// Unique tag identifier for the filesystem
+    pub tag: String,
+    /// Whether the filesystem is read-only
+    /// TODO: change libvirt to detect this
+    #[allow(dead_code)]
+    pub readonly: bool,
+}
+
 /// Builder for creating libvirt domain XML configurations
 #[derive(Debug)]
 pub struct DomainBuilder {
@@ -23,6 +36,7 @@ pub struct DomainBuilder {
     kernel_args: Option<String>,
     metadata: HashMap<String, String>,
     qemu_args: Vec<String>,
+    virtiofs_filesystems: Vec<VirtiofsFilesystem>,
 }
 
 impl Default for DomainBuilder {
@@ -45,6 +59,7 @@ impl DomainBuilder {
             kernel_args: None,
             metadata: HashMap::new(),
             qemu_args: Vec::new(),
+            virtiofs_filesystems: Vec::new(),
         }
     }
 
@@ -102,6 +117,12 @@ impl DomainBuilder {
         self
     }
 
+    /// Add a virtiofs filesystem mount
+    pub fn with_virtiofs_filesystem(mut self, filesystem: VirtiofsFilesystem) -> Self {
+        self.virtiofs_filesystems.push(filesystem);
+        self
+    }
+
     /// Build the domain XML
     pub fn build_xml(self) -> Result<String> {
         let name = self.name.ok_or_else(|| eyre!("Domain name is required"))?;
@@ -165,6 +186,15 @@ impl DomainBuilder {
 
         xml.push_str("\n  </os>");
 
+        // Add memory backing for shared memory support (required for virtiofs)
+        xml.push_str(
+            r#"
+  <memoryBacking>
+    <source type="memfd"/>
+    <access mode="shared"/>
+  </memoryBacking>"#,
+        );
+
         // Architecture-specific features
         xml.push_str(arch_config.xml_features());
 
@@ -277,6 +307,29 @@ impl DomainBuilder {
             ));
         }
 
+        // Virtiofs filesystems
+        for filesystem in &self.virtiofs_filesystems {
+            xml.push_str(&format!(
+                r#"
+    <filesystem type="mount" accessmode="passthrough">
+      <driver type="virtiofs" queue="1024"/>
+      <source dir="{}"/>
+      <target dir="{}"/>"#,
+                filesystem.source_dir, filesystem.tag
+            ));
+            // TODO: detect libvirt version for this
+            //         if filesystem.readonly {
+            //             xml.push_str(
+            //                 r#"
+            //   <readonly/>"#,
+            //             );
+            //         }
+            xml.push_str(
+                r#"
+    </filesystem>"#,
+            );
+        }
+
         xml.push_str("\n  </devices>");
 
         // QEMU commandline section (if we have QEMU args)
diff --git a/crates/kit/src/libvirt/run.rs b/crates/kit/src/libvirt/run.rs
@@ -12,6 +12,7 @@ use std::hash::{Hash, Hasher};
 
 use crate::common_opts::MemoryOpts;
 use crate::domain_list::DomainLister;
+use crate::libvirt::domain::VirtiofsFilesystem;
 use crate::utils::parse_memory_to_mb;
 
 /// Options for creating and running a bootable container VM
@@ -58,6 +59,10 @@ pub struct LibvirtRunOpts {
     /// Automatically SSH into the VM after creation
     #[clap(long)]
     pub ssh: bool,
+
+    /// Mount host container storage (RO) at /run/virtiofs-mnt-hoststorage
+    #[clap(long = "bind-storage-ro")]
+    pub bind_storage_ro: bool,
 }
 
 /// Execute the libvirt run command
@@ -376,7 +381,7 @@ fn create_libvirt_domain_from_disk(
     let memory = parse_memory_to_mb(&opts.memory.memory)?;
 
     // Build domain XML using the existing DomainBuilder with bootc metadata and SSH keys
-    let domain_xml = DomainBuilder::new()
+    let mut domain_builder = DomainBuilder::new()
         .with_name(domain_name)
         .with_memory(memory.into())
         .with_vcpus(opts.cpus)
@@ -390,7 +395,33 @@ fn create_libvirt_domain_from_disk(
         .with_metadata("bootc:network", &opts.network)
         .with_metadata("bootc:ssh-generated", "true")
         .with_metadata("bootc:ssh-private-key-base64", &private_key_base64)
-        .with_metadata("bootc:ssh-port", &ssh_port.to_string())
+        .with_metadata("bootc:ssh-port", &ssh_port.to_string());
+
+    // Add container storage mount if requested
+    if opts.bind_storage_ro {
+        let storage_path = crate::utils::detect_container_storage_path()
+            .context("Failed to detect container storage path.")?;
+        crate::utils::validate_container_storage_path(&storage_path)
+            .context("Container storage validation failed")?;
+
+        debug!(
+            "Adding container storage from {} as hoststorage virtiofs mount",
+            storage_path
+        );
+
+        let virtiofs_fs = VirtiofsFilesystem {
+            source_dir: storage_path.to_string(),
+            tag: "hoststorage".to_string(),
+            readonly: true,
+        };
+
+        domain_builder = domain_builder
+            .with_virtiofs_filesystem(virtiofs_fs)
+            .with_metadata("bootc:bind-storage-ro", "true")
+            .with_metadata("bootc:storage-path", storage_path.as_str());
+    }
+
+    let domain_xml = domain_builder
         .with_qemu_args(vec![
             "-smbios".to_string(),
             format!("type=11,value={}", smbios_cred),
diff --git a/docs/src/libvirt-run.md b/docs/src/libvirt-run.md
@@ -131,6 +131,34 @@ bcvk libvirt run \
 - Persistent development state
 - Host integration capabilities
 
+### Container Storage Integration
+
+```bash
+# Create VM with access to host container storage for bootc upgrades
+bcvk libvirt run \
+  --name upgrade-test \
+  --bind-storage-ro \
+  --ssh \
+  quay.io/fedora/fedora-bootc:42
+```
+
+With this, a virtiofs mount named `hoststorage` is provisioned. There isn't
+yet automatic mounting, but you can inject code to do so that performs
+`mkdir /run/hoststorage && mount -t virtiofs hoststorage /run/hoststorage`.
+
+Then on your host system after you've done a `podman build` that results in a new image `localhost/bootc`,
+in the guest system you can point bootc to use it via e.g.
+```
+env STORAGE_OPTS=additionalimagestore=/run/hoststorage bootc switch --transport containers-storage localhost/bootc
+```
+
+You currently need to add the `STORAGE_OPTS` each time you invoke `bootc` - but there after e.g.
+```
+env STORAGE_OPTS=additionalimagestore=/run/hoststorage bootc upgrade
+```
+
+will work.
+
 ## Resource Management Concepts
 
 ### CPU Allocation
diff --git a/docs/src/man/bcvk-libvirt-run.md b/docs/src/man/bcvk-libvirt-run.md
