diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f5da914258..a0f43750992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to ### Added +- [#5774](https://github.com/firecracker-microvm/firecracker/pull/5774): Add + support for block device path overriding on snapshot restore. - [#5323](https://github.com/firecracker-microvm/firecracker/pull/5323): Add support for Vsock Unix domain socket path overriding on snapshot restore. More information can be found in the diff --git a/docs/snapshotting/network-for-clones.md b/docs/snapshotting/network-for-clones.md index 63115418765..2c46a3e063c 100644 --- a/docs/snapshotting/network-for-clones.md +++ b/docs/snapshotting/network-for-clones.md @@ -200,6 +200,37 @@ ip addr add 172.16.3.2/30 dev eth0 ip route add default via 172.16.3.1/30 dev eth0 ``` +### Overriding block device paths + +When restoring a VM from a snapshot on a different host or in an environment +where disk paths are non-deterministic (e.g. container runtimes using +devmapper), the block device paths baked into the snapshot state may no longer +be valid. In this case you can use the `drive_overrides` parameter of the +snapshot restore API to specify the new host path for each block device. + +For example, if we have a block device with drive ID `rootfs` in the snapshotted +microVM, we can override its path during snapshot resume: + +```bash +curl --unix-socket /tmp/firecracker.socket -i \ + -X PUT 'http://localhost/snapshot/load' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "snapshot_path": "./snapshot_file", + "mem_backend": { + "backend_path": "./mem_file", + "backend_type": "File" + }, + "drive_overrides": [ + { + "drive_id": "rootfs", + "path_on_host": "/new/path/to/rootfs.ext4" + } + ] + }' +``` + # Ingress connectivity The above setup only provides egress connectivity. If in addition we also want diff --git a/src/firecracker/src/api_server/request/snapshot.rs b/src/firecracker/src/api_server/request/snapshot.rs index 04c20de6b2d..43dc5d3876f 100644 --- a/src/firecracker/src/api_server/request/snapshot.rs +++ b/src/firecracker/src/api_server/request/snapshot.rs @@ -111,6 +111,7 @@ fn parse_put_snapshot_load(body: &Body) -> Result { resume_vm: snapshot_config.resume_vm, network_overrides: snapshot_config.network_overrides, vsock_override: snapshot_config.vsock_override, + drive_overrides: snapshot_config.drive_overrides, }; // Construct the `ParsedRequest` object. @@ -126,7 +127,9 @@ fn parse_put_snapshot_load(body: &Body) -> Result { #[cfg(test)] mod tests { - use vmm::vmm_config::snapshot::{MemBackendConfig, MemBackendType, NetworkOverride}; + use vmm::vmm_config::snapshot::{ + DriveOverride, MemBackendConfig, MemBackendType, NetworkOverride, + }; use super::*; use crate::api_server::parsed_request::tests::{depr_action_from_req, vmm_action_from_request}; @@ -189,6 +192,7 @@ mod tests { resume_vm: false, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }; let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); assert!( @@ -220,6 +224,7 @@ mod tests { resume_vm: false, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }; let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); assert!( @@ -251,6 +256,7 @@ mod tests { resume_vm: true, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }; let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); assert!( @@ -291,6 +297,48 @@ mod tests { host_dev_name: String::from("vmtap2"), }], vsock_override: None, + drive_overrides: vec![], + }; + let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); + assert!( + parsed_request + .parsing_info() + .take_deprecation_message() + .is_none() + ); + assert_eq!( + vmm_action_from_request(parsed_request), + VmmAction::LoadSnapshot(expected_config) + ); + + let body = r#"{ + "snapshot_path": "foo", + "mem_backend": { + "backend_path": "bar", + "backend_type": "File" + }, + "resume_vm": true, + "drive_overrides": [ + { + "drive_id": "rootfs", + "path_on_host": "/new/path/rootfs.ext4" + } + ] + }"#; + let expected_config = LoadSnapshotParams { + snapshot_path: PathBuf::from("foo"), + mem_backend: MemBackendConfig { + backend_path: PathBuf::from("bar"), + backend_type: MemBackendType::File, + }, + track_dirty_pages: false, + resume_vm: true, + network_overrides: vec![], + vsock_override: None, + drive_overrides: vec![DriveOverride { + drive_id: String::from("rootfs"), + path_on_host: String::from("/new/path/rootfs.ext4"), + }], }; let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); assert!( @@ -319,6 +367,7 @@ mod tests { resume_vm: true, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }; let parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap(); assert_eq!( diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index 8d87eefc11a..b4ea8e53786 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -1603,6 +1603,24 @@ definitions: description: The new path for the backing Unix Domain Socket. + DriveOverride: + type: object + description: + Allows for changing the backing host path of a block device + during snapshot restore. + required: + - drive_id + - path_on_host + properties: + drive_id: + type: string + description: + The ID of the drive to modify + path_on_host: + type: string + description: + The new path on the host for the block device's backing file + SnapshotLoadParams: type: object description: @@ -1650,6 +1668,11 @@ definitions: for restoring a snapshot with a different socket path than the one used when the snapshot was created. For example, when the original socket path is no longer available or when deploying to a different environment. + drive_overrides: + type: array + description: Block device host paths to override + items: + $ref: "#/definitions/DriveOverride" TokenBucket: diff --git a/src/vmm/src/devices/virtio/block/persist.rs b/src/vmm/src/devices/virtio/block/persist.rs index cb9a6471137..e94e022c9bb 100644 --- a/src/vmm/src/devices/virtio/block/persist.rs +++ b/src/vmm/src/devices/virtio/block/persist.rs @@ -20,8 +20,24 @@ pub enum BlockState { impl BlockState { pub fn is_activated(&self) -> bool { match self { - BlockState::Virtio(virtio_block_state) => virtio_block_state.virtio_state.activated, - BlockState::VhostUser(vhost_user_block_state) => false, + BlockState::Virtio(state) => state.virtio_state.activated, + BlockState::VhostUser(_) => false, + } + } + + /// Returns the drive ID. + pub fn id(&self) -> &str { + match self { + BlockState::Virtio(state) => &state.id, + BlockState::VhostUser(state) => &state.id, + } + } + + /// Overrides the host path (disk path or socket path) of the block device. + pub fn set_host_path(&mut self, path: &str) { + match self { + BlockState::Virtio(state) => state.disk_path = path.to_string(), + BlockState::VhostUser(state) => state.socket_path = path.to_string(), } } } @@ -31,3 +47,90 @@ impl BlockState { pub struct BlockConstructorArgs { pub mem: GuestMemoryMmap, } + +#[cfg(test)] +mod tests { + use super::*; + + fn virtio_block_state(id: &str, disk_path: &str, activated: bool) -> VirtioBlockState { + serde_json::from_value(serde_json::json!({ + "id": id, + "partuuid": null, + "cache_type": "Unsafe", + "root_device": false, + "disk_path": disk_path, + "virtio_state": { + "device_type": "Block", + "avail_features": 0, + "acked_features": 0, + "queues": [], + "activated": activated + }, + "rate_limiter_state": { + "ops": null, + "bandwidth": null + }, + "file_engine_type": "Sync" + })) + .unwrap() + } + + fn vhost_user_block_state(id: &str, socket_path: &str) -> VhostUserBlockState { + serde_json::from_value(serde_json::json!({ + "id": id, + "partuuid": null, + "cache_type": "Unsafe", + "root_device": false, + "socket_path": socket_path, + "vu_acked_protocol_features": 0, + "config_space": [], + "virtio_state": { + "device_type": "Block", + "avail_features": 0, + "acked_features": 0, + "queues": [], + "activated": false + } + })) + .unwrap() + } + + #[test] + fn test_block_state_id() { + let virtio = BlockState::Virtio(virtio_block_state("rootfs", "/path", false)); + assert_eq!(virtio.id(), "rootfs"); + + let vhost = BlockState::VhostUser(vhost_user_block_state("scratch", "/sock")); + assert_eq!(vhost.id(), "scratch"); + } + + #[test] + fn test_block_state_is_activated() { + let active = BlockState::Virtio(virtio_block_state("rootfs", "/path", true)); + assert!(active.is_activated()); + + let inactive = BlockState::Virtio(virtio_block_state("rootfs", "/path", false)); + assert!(!inactive.is_activated()); + + // vhost-user always returns false + let vhost = BlockState::VhostUser(vhost_user_block_state("rootfs", "/sock")); + assert!(!vhost.is_activated()); + } + + #[test] + fn test_block_state_set_host_path() { + let mut virtio = BlockState::Virtio(virtio_block_state("rootfs", "/old/path", false)); + virtio.set_host_path("/new/path"); + match &virtio { + BlockState::Virtio(state) => assert_eq!(state.disk_path, "/new/path"), + _ => panic!("expected Virtio variant"), + } + + let mut vhost = BlockState::VhostUser(vhost_user_block_state("rootfs", "/old/sock")); + vhost.set_host_path("/new/sock"); + match &vhost { + BlockState::VhostUser(state) => assert_eq!(state.socket_path, "/new/sock"), + _ => panic!("expected VhostUser variant"), + } + } +} diff --git a/src/vmm/src/devices/virtio/block/vhost_user/persist.rs b/src/vmm/src/devices/virtio/block/vhost_user/persist.rs index d507fa9577b..39aa439e86b 100644 --- a/src/vmm/src/devices/virtio/block/vhost_user/persist.rs +++ b/src/vmm/src/devices/virtio/block/vhost_user/persist.rs @@ -15,11 +15,11 @@ use crate::snapshot::Persist; /// vhost-user block device state. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VhostUserBlockState { - id: String, + pub id: String, partuuid: Option, cache_type: CacheType, root_device: bool, - socket_path: String, + pub socket_path: String, vu_acked_protocol_features: u64, config_space: Vec, virtio_state: VirtioDeviceState, diff --git a/src/vmm/src/devices/virtio/block/virtio/persist.rs b/src/vmm/src/devices/virtio/block/virtio/persist.rs index c4288460a56..cd6bd457bd4 100644 --- a/src/vmm/src/devices/virtio/block/virtio/persist.rs +++ b/src/vmm/src/devices/virtio/block/virtio/persist.rs @@ -52,11 +52,11 @@ impl From for FileEngineType { /// Holds info about the block device. Gets saved in snapshot. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VirtioBlockState { - id: String, + pub id: String, partuuid: Option, cache_type: CacheType, root_device: bool, - disk_path: String, + pub disk_path: String, pub virtio_state: VirtioDeviceState, rate_limiter_state: RateLimiterState, file_engine_type: FileEngineTypeState, diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index 8f297bf0784..5a880fde4ff 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -408,6 +408,26 @@ pub fn restore_from_snapshot( .clone_from(&vsock_override.uds_path); } + for entry in ¶ms.drive_overrides { + microvm_state + .device_states + .mmio_state + .block_devices + .iter_mut() + .map(|device| &mut device.device_state) + .chain( + microvm_state + .device_states + .pci_state + .block_devices + .iter_mut() + .map(|device| &mut device.device_state), + ) + .find(|x| x.id() == entry.drive_id) + .map(|device_state| device_state.set_host_path(&entry.path_on_host)) + .ok_or(SnapshotStateFromFileError::UnknownBlockDevice)?; + } + let track_dirty_pages = params.track_dirty_pages; let vcpu_count = microvm_state @@ -481,6 +501,8 @@ pub enum SnapshotStateFromFileError { UnknownNetworkDevice, /// Unknown Vsock Device. UnknownVsockDevice, + /// Unknown Block Device. + UnknownBlockDevice, } fn snapshot_state_from_file( diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index 4617890a0e4..e6570ceec43 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -1286,6 +1286,7 @@ mod tests { resume_vm: false, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }, ))); check_unsupported(runtime_request(VmmAction::SetEntropyDevice( diff --git a/src/vmm/src/vmm_config/snapshot.rs b/src/vmm/src/vmm_config/snapshot.rs index 1da084887f3..998168b63c2 100644 --- a/src/vmm/src/vmm_config/snapshot.rs +++ b/src/vmm/src/vmm_config/snapshot.rs @@ -64,6 +64,16 @@ pub struct VsockOverride { pub uds_path: String, } +/// Allows for changing the backing host path of a block device +/// during snapshot restore +#[derive(Debug, PartialEq, Eq, Deserialize)] +pub struct DriveOverride { + /// The ID of the drive to modify + pub drive_id: String, + /// The new path to the backing file on the host + pub path_on_host: String, +} + /// Stores the configuration that will be used for loading a snapshot. #[derive(Debug, PartialEq, Eq)] pub struct LoadSnapshotParams { @@ -81,6 +91,8 @@ pub struct LoadSnapshotParams { pub network_overrides: Vec, /// When set, the vsock backend UDS path will be overridden pub vsock_override: Option, + /// The block devices to override on load. + pub drive_overrides: Vec, } /// Stores the configuration for loading a snapshot that is provided by the user. @@ -113,6 +125,9 @@ pub struct LoadSnapshotConfig { /// Whether or not to override the vsock backend UDS path. #[serde(skip_serializing_if = "Option::is_none")] pub vsock_override: Option, + /// The block devices to override on load. + #[serde(default)] + pub drive_overrides: Vec, } /// Stores the configuration used for managing snapshot memory. diff --git a/src/vmm/tests/integration_tests.rs b/src/vmm/tests/integration_tests.rs index 4d58b95a426..b7c81db38cc 100644 --- a/src/vmm/tests/integration_tests.rs +++ b/src/vmm/tests/integration_tests.rs @@ -296,6 +296,7 @@ fn verify_load_snapshot(snapshot_file: TempFile, memory_file: TempFile) { resume_vm: true, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], })) .unwrap(); @@ -381,6 +382,7 @@ fn verify_load_snap_disallowed_after_boot_resources(res: VmmAction, res_name: &s resume_vm: false, network_overrides: vec![], vsock_override: None, + drive_overrides: vec![], }); let err = preboot_api_controller.handle_preboot_request(req); assert!( diff --git a/tests/framework/microvm.py b/tests/framework/microvm.py index 023589fe2c9..997be19fa3e 100644 --- a/tests/framework/microvm.py +++ b/tests/framework/microvm.py @@ -1084,6 +1084,7 @@ def restore_from_snapshot( resume: bool = False, rename_interfaces: dict = None, vsock_override: str = None, + drive_overrides: list = None, *, uffd_handler_name: str = None, ): @@ -1143,6 +1144,9 @@ def restore_from_snapshot( if vsock_override is not None: optional_kwargs["vsock_override"] = {"uds_path": vsock_override} + if drive_overrides is not None: + optional_kwargs["drive_overrides"] = drive_overrides + self.api.snapshot_load.put( mem_backend=mem_backend, snapshot_path=str(jailed_vmstate), diff --git a/tests/integration_tests/functional/test_snapshot_basic.py b/tests/integration_tests/functional/test_snapshot_basic.py index 1883f837a1f..4a8e902ad4d 100644 --- a/tests/integration_tests/functional/test_snapshot_basic.py +++ b/tests/integration_tests/functional/test_snapshot_basic.py @@ -606,3 +606,58 @@ def test_snapshot_rename_vsock( restored_vm.spawn() restored_vm.restore_from_snapshot(snapshot, vsock_override="/v.sock2", resume=True) + + +def test_snapshot_override_drive(uvm_nano, microvm_factory): + """ + Test that we can restore a snapshot and point a block device to a + different host path. + """ + vm = uvm_nano + vm.add_net_iface() + vm.start() + + snapshot = vm.snapshot_full() + + restored_vm = microvm_factory.build() + restored_vm.spawn() + + # Copy the rootfs to a new file inside the restored VM's chroot so + # Firecracker (which runs jailed) can access it. Use a different file + # (and inode) than the original to verify the override actually works. + original_rootfs = snapshot.disks["rootfs"] + shutil.copy(original_rootfs, Path(restored_vm.chroot()) / "rootfs_override.ext4") + + restored_vm.restore_from_snapshot( + snapshot, + drive_overrides=[ + {"drive_id": "rootfs", "path_on_host": "/rootfs_override.ext4"}, + ], + resume=True, + ) + + +def test_drive_override_fails_unknown_id(uvm_nano, microvm_factory): + """ + Providing a drive override with an unknown drive_id should fail. + """ + vm = uvm_nano + vm.start() + + snapshot = vm.snapshot_full() + vm.kill() + + restored_vm = microvm_factory.build() + restored_vm.spawn() + + # The failed snapshot load causes Firecracker to exit. + with pytest.raises(RuntimeError, match="Unknown Block Device"): + restored_vm.restore_from_snapshot( + snapshot, + drive_overrides=[ + {"drive_id": "nonexistent", "path_on_host": "/fake/path"}, + ], + resume=True, + ) + + restored_vm.mark_killed()