From e0fe29d01e1f32ebeca9025026d2b1676fad1468 Mon Sep 17 00:00:00 2001 From: danbugs Date: Tue, 12 May 2026 02:42:40 +0000 Subject: [PATCH 1/3] sparsify snapshot: punch zero pages to reduce disk usage by ~80% MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After writing the HLS snapshot, scan for all-zero 4 KiB pages and punch them with fallocate(PUNCH_HOLE). This makes the file sparse: zeros still read back via mmap but consume no disk blocks. On a typical pyhl snapshot (Python + pandas/numpy warm): Before: 2016 MiB on disk After: 648 MiB on disk (same 2016 MiB apparent size) No load-time overhead — mmap of sparse files is identical to dense. Linux-only; other platforms get a no-op fallback. Signed-off-by: danbugs --- host/Cargo.lock | 1 + host/Cargo.toml | 1 + host/src/bin/pyhl.rs | 16 +++++++++- host/src/lib.rs | 73 +++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/host/Cargo.lock b/host/Cargo.lock index 93e75a5..181e3a6 100644 --- a/host/Cargo.lock +++ b/host/Cargo.lock @@ -575,6 +575,7 @@ dependencies = [ "base64", "clap", "hyperlight-host", + "libc", "memmap2", "nix", "serde_json", diff --git a/host/Cargo.toml b/host/Cargo.toml index 3f40286..b2a1719 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -38,4 +38,5 @@ base64 = "0.22" [target.'cfg(unix)'.dependencies] nix = { version = "0.29", features = ["fs"] } +libc = "0.2" diff --git a/host/src/bin/pyhl.rs b/host/src/bin/pyhl.rs index 685ff8d..8cca148 100644 --- a/host/src/bin/pyhl.rs +++ b/host/src/bin/pyhl.rs @@ -373,8 +373,9 @@ fn cmd_setup(args: SetupArgs) -> Result<()> { mib(&dst_initrd) ); eprintln!( - " snapshot: {} ({} MiB)", + " snapshot: {} ({} MiB on disk, {} MiB apparent)", dst_snapshot.display(), + disk_mib(&dst_snapshot), mib(&dst_snapshot) ); Ok(()) @@ -384,6 +385,19 @@ fn mib(p: &Path) -> u64 { fs::metadata(p).map(|m| m.len() / 1024 / 1024).unwrap_or(0) } +#[cfg(unix)] +fn disk_mib(p: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + fs::metadata(p) + .map(|m| m.blocks() * 512 / 1024 / 1024) + .unwrap_or_else(|_| mib(p)) +} + +#[cfg(not(unix))] +fn disk_mib(p: &Path) -> u64 { + mib(p) +} + /// Lightweight timestamp (seconds since epoch in ISO-8601-ish) so we don't /// need to pull chrono just for the VERSION stamp. fn now_iso8601() -> String { diff --git a/host/src/lib.rs b/host/src/lib.rs index e953a6f..b1335a8 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -1355,17 +1355,19 @@ impl Sandbox { Ok(()) } - /// Persist the current post-evolve (or post-`snapshot_now`) snapshot - /// to disk so a later process can skip evolve + init and go straight - /// to `call`. Uses hyperlight's `Snapshot::to_file` — the file - /// format and cross-platform mmap load are documented in - /// hyperlight/docs/snapshot-file-implementation-plan.md. + /// Persist the current snapshot to disk as a sparse file. + /// + /// After writing the raw HLS snapshot, zero-filled 4 KiB pages are + /// punched out with `fallocate(PUNCH_HOLE)` so they consume no disk + /// space. The file is still mmap-loadable — holes read back as + /// zeros with no decompression overhead. pub fn save_snapshot>(&self, path: P) -> Result<()> { let snap = self .snapshot .as_ref() .ok_or_else(|| anyhow!("no snapshot present; build() or snapshot_now() first"))?; snap.to_file(path.as_ref())?; + sparsify_snapshot(path.as_ref())?; Ok(()) } @@ -1530,6 +1532,67 @@ pub fn run_vm_capture_output( }) } +// --------------------------------------------------------------------------- +// Snapshot sparsification +// --------------------------------------------------------------------------- + +/// Punch holes in zero-filled 4 KiB pages of a snapshot file. +/// +/// The HLS snapshot format is a 4 KiB header followed by a dense memory +/// blob where ~80 % of pages are all-zeros (unused heap). Punching them +/// with `fallocate(PUNCH_HOLE)` turns the file sparse — the zeros still +/// read back via mmap but consume no disk blocks. +#[cfg(target_os = "linux")] +fn sparsify_snapshot(path: &Path) -> Result<()> { + use std::os::unix::io::AsRawFd; + + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path)?; + let len = file.metadata()?.len(); + let mmap = unsafe { memmap2::Mmap::map(&file)? }; + + const PAGE: usize = 4096; + const HEADER: usize = PAGE; + let zero_page = [0u8; PAGE]; + + let mut punched = 0u64; + let mut offset = HEADER; + while offset + PAGE <= len as usize { + if mmap[offset..offset + PAGE] == zero_page { + let ret = unsafe { + libc::fallocate( + file.as_raw_fd(), + libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE, + offset as i64, + PAGE as i64, + ) + }; + if ret == 0 { + punched += 1; + } + } + offset += PAGE; + } + drop(mmap); + + if punched > 0 { + eprintln!( + " sparsified: punched {} zero pages ({} MiB saved on disk)", + punched, + punched * 4 / 1024 + ); + } + + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn sparsify_snapshot(_path: &Path) -> Result<()> { + Ok(()) +} + // --------------------------------------------------------------------------- // FsSandbox tests — prove that host-side path resolution rejects escapes. // From 8cb5a38aba68ad08c5b183f82ffcb71def3ab3df Mon Sep 17 00:00:00 2001 From: danbugs Date: Tue, 12 May 2026 02:50:10 +0000 Subject: [PATCH 2/3] add Windows sparsify_snapshot via FSCTL_SET_SPARSE + FSCTL_SET_ZERO_DATA Signed-off-by: danbugs --- host/Cargo.lock | 1 + host/Cargo.toml | 3 ++ host/src/bin/pyhl.rs | 21 +++++++++- host/src/lib.rs | 93 +++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 116 insertions(+), 2 deletions(-) diff --git a/host/Cargo.lock b/host/Cargo.lock index 181e3a6..ed10cd1 100644 --- a/host/Cargo.lock +++ b/host/Cargo.lock @@ -579,6 +579,7 @@ dependencies = [ "memmap2", "nix", "serde_json", + "windows-sys", ] [[package]] diff --git a/host/Cargo.toml b/host/Cargo.toml index b2a1719..17a3dc8 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -40,3 +40,6 @@ base64 = "0.22" nix = { version = "0.29", features = ["fs"] } libc = "0.2" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_System_IO", "Win32_Storage_FileSystem"] } + diff --git a/host/src/bin/pyhl.rs b/host/src/bin/pyhl.rs index 8cca148..49c1ec1 100644 --- a/host/src/bin/pyhl.rs +++ b/host/src/bin/pyhl.rs @@ -393,7 +393,26 @@ fn disk_mib(p: &Path) -> u64 { .unwrap_or_else(|_| mib(p)) } -#[cfg(not(unix))] +#[cfg(windows)] +fn disk_mib(p: &Path) -> u64 { + use std::os::windows::ffi::OsStrExt; + let wide: Vec = p + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut high: u32 = 0; + let low = unsafe { + windows_sys::Win32::Storage::FileSystem::GetCompressedFileSizeW(wide.as_ptr(), &mut high) + }; + if low == u32::MAX { + return mib(p); + } + let bytes = ((high as u64) << 32) | (low as u64); + bytes / 1024 / 1024 +} + +#[cfg(not(any(unix, windows)))] fn disk_mib(p: &Path) -> u64 { mib(p) } diff --git a/host/src/lib.rs b/host/src/lib.rs index b1335a8..3edf1fa 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -1588,7 +1588,98 @@ fn sparsify_snapshot(path: &Path) -> Result<()> { Ok(()) } -#[cfg(not(target_os = "linux"))] +/// Windows equivalent: mark the file sparse with FSCTL_SET_SPARSE, then +/// punch zero ranges with FSCTL_SET_ZERO_DATA. +#[cfg(target_os = "windows")] +fn sparsify_snapshot(path: &Path) -> Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{FSCTL_SET_SPARSE, FSCTL_SET_ZERO_DATA}; + use windows_sys::Win32::System::IO::DeviceIoControl; + + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path)?; + let len = file.metadata()?.len(); + let handle = file.as_raw_handle() as isize; + + // Mark file as sparse. + let ok = unsafe { + DeviceIoControl( + handle, + FSCTL_SET_SPARSE, + std::ptr::null(), + 0, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if ok == 0 { + return Ok(()); + } + + let mmap = unsafe { memmap2::Mmap::map(&file)? }; + + const PAGE: usize = 4096; + const HEADER: usize = PAGE; + let zero_page = [0u8; PAGE]; + + // Coalesce contiguous zero pages into ranges for fewer syscalls. + let mut punched = 0u64; + let mut offset = HEADER; + while offset + PAGE <= len as usize { + if mmap[offset..offset + PAGE] != zero_page { + offset += PAGE; + continue; + } + let range_start = offset; + while offset + PAGE <= len as usize && mmap[offset..offset + PAGE] == zero_page { + offset += PAGE; + } + let range_end = offset; + + #[repr(C)] + struct FileZeroDataInformation { + file_offset: i64, + beyond_final_zero: i64, + } + + let info = FileZeroDataInformation { + file_offset: range_start as i64, + beyond_final_zero: range_end as i64, + }; + let ok = unsafe { + DeviceIoControl( + handle, + FSCTL_SET_ZERO_DATA, + &info as *const _ as *const _, + std::mem::size_of::() as u32, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if ok != 0 { + punched += (range_end - range_start) as u64 / PAGE as u64; + } + } + drop(mmap); + + if punched > 0 { + eprintln!( + " sparsified: punched {} zero pages ({} MiB saved on disk)", + punched, + punched * 4 / 1024 + ); + } + + Ok(()) +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] fn sparsify_snapshot(_path: &Path) -> Result<()> { Ok(()) } From d61233784cd3587d80201f7a54e9661069473b38 Mon Sep 17 00:00:00 2001 From: danbugs Date: Tue, 12 May 2026 02:51:30 +0000 Subject: [PATCH 3/3] fix Windows sparsify: use Win32_System_Ioctl for FSCTL constants, fix HANDLE type Signed-off-by: danbugs --- host/Cargo.toml | 2 +- host/src/lib.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/host/Cargo.toml b/host/Cargo.toml index 17a3dc8..8711d5b 100644 --- a/host/Cargo.toml +++ b/host/Cargo.toml @@ -41,5 +41,5 @@ nix = { version = "0.29", features = ["fs"] } libc = "0.2" [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_System_IO", "Win32_Storage_FileSystem"] } +windows-sys = { version = "0.61", features = ["Win32_System_IO", "Win32_System_Ioctl", "Win32_Storage_FileSystem"] } diff --git a/host/src/lib.rs b/host/src/lib.rs index 3edf1fa..90976d2 100644 --- a/host/src/lib.rs +++ b/host/src/lib.rs @@ -1593,7 +1593,7 @@ fn sparsify_snapshot(path: &Path) -> Result<()> { #[cfg(target_os = "windows")] fn sparsify_snapshot(path: &Path) -> Result<()> { use std::os::windows::io::AsRawHandle; - use windows_sys::Win32::Storage::FileSystem::{FSCTL_SET_SPARSE, FSCTL_SET_ZERO_DATA}; + use windows_sys::Win32::System::Ioctl::{FSCTL_SET_SPARSE, FSCTL_SET_ZERO_DATA}; use windows_sys::Win32::System::IO::DeviceIoControl; let file = std::fs::OpenOptions::new() @@ -1601,7 +1601,7 @@ fn sparsify_snapshot(path: &Path) -> Result<()> { .write(true) .open(path)?; let len = file.metadata()?.len(); - let handle = file.as_raw_handle() as isize; + let handle = file.as_raw_handle(); // Mark file as sparse. let ok = unsafe {