openvmm: add cold-plug VFIO PCI pass-through support

Cold-plug VFIO devices (GPUs, NVSwitches, InfiniBand VFs) are now wired into the OpenVMM Config before launch:

- Reserve 16 PCIe root ports named vfio0..vfio15 with hotplug=false in the root complex.

- In start_vm, process pending DeviceType::Vfio by opening /dev/vfio/<group>, building a VfioDeviceHandle { pci_id, group } per HostDevice in the IOMMU group, and pushing a PcieDeviceConfig on a pre-reserved port.

- Reject post-launch VFIO add_device with an explicit cold-plug-only error.

- Add vfio_assigned_device_resources git dep at the existing openvmm rev.

Author: JocelynBerrendonner

Cargo.toml

@@ -118,6 +118,7 @@ virtio_resources = { git = "https://github.com/microsoft/openvmm", rev = "14ad21
 vm_resource = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592" }
 disk_backend_resources = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592" }
 disk_blockdevice = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592" }
+vfio_assigned_device_resources = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592" }
 ovmm_vmm_core_defs = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592", package = "vmm_core_defs" }
 ovmm_guid = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592", package = "guid" }
 ovmm_memory_range = { git = "https://github.com/microsoft/openvmm", rev = "14ad213b739a2ca7976f00ea090b475ceddd4592", package = "memory_range" }

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -88,6 +88,7 @@ virtio_resources = { workspace = true, optional = true }
 vm_resource = { workspace = true, optional = true }
 disk_backend_resources = { workspace = true, optional = true }
 disk_blockdevice = { workspace = true, optional = true }
+vfio_assigned_device_resources = { workspace = true, optional = true }
 ovmm_vmm_core_defs = { workspace = true, optional = true }
 ovmm_guid = { workspace = true, optional = true }
 ovmm_memory_range = { workspace = true, optional = true }
@@ -123,6 +124,7 @@ openvmm = [
     "dep:vm_resource",
     "dep:disk_backend_resources",
     "dep:disk_blockdevice",
+    "dep:vfio_assigned_device_resources",
     "dep:ovmm_vmm_core_defs",
     "dep:ovmm_guid",
     "dep:ovmm_memory_range",

src/runtime-rs/crates/hypervisor/src/openvmm/inner_device.rs

@@ -117,6 +117,13 @@ impl OpenVmmInner {
                 Ok(DeviceType::Block(block))
             }
             other => {
+                if matches!(other, DeviceType::Vfio(_)) {
+                    return Err(anyhow!(
+                        "openvmm: VFIO devices are cold-plug only and must be \
+                         added before start_vm; got {} after VMM start",
+                        other
+                    ));
+                }
                 warn!(sl!(), "openvmm: add_device stub for {}", other);
                 Ok(other)
             }

src/runtime-rs/crates/hypervisor/src/openvmm/inner_hypervisor.rs

@@ -16,6 +16,7 @@ use super::vmm_instance::DeferredNetworkDevice;
 use super::{
     OPENVMM_BLOCK_HOTPLUG_PORT_COUNT, OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX, OPENVMM_CONSOLE_PCI_PORT,
     OPENVMM_NET_PCI_PORT, OPENVMM_ROOTFS_PCI_PORT, OPENVMM_SHAREFS_PCI_PORT,
+    OPENVMM_VFIO_COLDPLUG_PORT_COUNT, OPENVMM_VFIO_COLDPLUG_PORT_PREFIX,
 };
 use crate::kernel_param::KernelParams;
 use crate::utils::{get_jailer_root, get_sandbox_path};
@@ -194,6 +195,17 @@ impl OpenVmmInner {
                     });
                 }
 
+                // Cold-plug ports for VFIO PCI pass-through (GPUs, NVSwitches,
+                // InfiniBand VFs). These are always created so the OpenVMM
+                // root-complex layout is stable; unused ones simply appear
+                // empty in the guest.
+                for index in 0..OPENVMM_VFIO_COLDPLUG_PORT_COUNT {
+                    ports.push(PcieRootPortConfig {
+                        name: format!("{}{}", OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, index),
+                        hotplug: false,
+                    });
+                }
+
                 ports
             },
         }];
@@ -221,6 +233,7 @@ impl OpenVmmInner {
             Vec::new();
         let mut deferred_block_devices = Vec::new();
         let mut deferred_network_devices = Vec::new();
+        let mut next_vfio_port: u8 = 0;
 
         for dev in &pending {
             match dev {
@@ -338,6 +351,72 @@ impl OpenVmmInner {
                         deferred_block_devices.push(dev.clone());
                     }
                 }
+                crate::DeviceType::Vfio(vfio_dev) => {
+                    // Cold-plug VFIO PCI pass-through. Each HostDevice in the
+                    // IOMMU group becomes its own PcieDeviceConfig on a
+                    // pre-reserved root port. The /dev/vfio/<group> fd is
+                    // opened here (one open per device, which the kernel
+                    // allows for the same group).
+                    let host_path = &vfio_dev.config.host_path;
+                    info!(
+                        sl!(),
+                        "openvmm: cold-plug VFIO group {} ({} device(s))",
+                        host_path,
+                        vfio_dev.devices.len()
+                    );
+
+                    for hostdev in &vfio_dev.devices {
+                        if hostdev.bus_slot_func.is_empty() {
+                            warn!(
+                                sl!(),
+                                "openvmm: skipping VFIO device with empty BDF in group {}",
+                                host_path
+                            );
+                            continue;
+                        }
+
+                        if next_vfio_port >= OPENVMM_VFIO_COLDPLUG_PORT_COUNT {
+                            return Err(anyhow!(
+                                "openvmm: too many VFIO devices (limit {}), cannot cold-plug BDF {}",
+                                OPENVMM_VFIO_COLDPLUG_PORT_COUNT,
+                                hostdev.bus_slot_func
+                            ));
+                        }
+
+                        let group_fd = std::fs::OpenOptions::new()
+                            .read(true)
+                            .write(true)
+                            .open(host_path)
+                            .with_context(|| {
+                                format!(
+                                    "openvmm: failed to open VFIO group {} for BDF {}",
+                                    host_path, hostdev.bus_slot_func
+                                )
+                            })?;
+
+                        let port_name = format!(
+                            "{}{}",
+                            OPENVMM_VFIO_COLDPLUG_PORT_PREFIX, next_vfio_port
+                        );
+                        next_vfio_port += 1;
+
+                        info!(
+                            sl!(),
+                            "openvmm: assigning VFIO BDF {} to port {}",
+                            hostdev.bus_slot_func,
+                            port_name
+                        );
+
+                        pcie_devices.push(PcieDeviceConfig {
+                            port_name,
+                            resource: vfio_assigned_device_resources::VfioDeviceHandle {
+                                pci_id: hostdev.bus_slot_func.clone(),
+                                group: group_fd,
+                            }
+                            .into_resource(),
+                        });
+                    }
+                }
                 other => {
                     warn!(sl!(), "openvmm: unsupported pending device type: {}", other);
                 }

src/runtime-rs/crates/hypervisor/src/openvmm/mod.rs

@@ -38,6 +38,11 @@ pub(crate) const OPENVMM_CONSOLE_PCI_PORT: &str = "rp4";
 pub(crate) const OPENVMM_STATIC_PCI_PORT_COUNT: u8 = 5;
 pub(crate) const OPENVMM_BLOCK_HOTPLUG_PORT_PREFIX: &str = "hp";
 pub(crate) const OPENVMM_BLOCK_HOTPLUG_PORT_COUNT: u8 = 24;
+/// Number of pre-reserved PCIe root ports for cold-plug VFIO pass-through
+/// devices (e.g., GPUs, NVSwitches). These ports are created with
+/// `hotplug: false` so the guest sees the assigned devices at boot.
+pub(crate) const OPENVMM_VFIO_COLDPLUG_PORT_PREFIX: &str = "vfio";
+pub(crate) const OPENVMM_VFIO_COLDPLUG_PORT_COUNT: u8 = 16;
 
 /// The OpenVMM hypervisor struct, wrapping inner state behind a lock.
 pub struct OpenVmm {