From 1b33671227420410923a02d1511c01ddef8355c7 Mon Sep 17 00:00:00 2001 From: Daniel Viktorovich Kamyshan Date: Tue, 19 May 2026 16:51:06 +0900 Subject: [PATCH] fsutil: add NFS soft-mount options to prevent kernel panic on hot-unplug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When user physically disconnects USB-C cable from an anylinuxfs-managed device without running `anylinuxfs unmount` first, macOS NFS client (default hard mount semantics) retries indefinitely against the now-unreachable NFS server inside libkrun. The kernel holds `IOMediaBSDClient` in busy state until `watchdogd` triggers `panic(busy timeout[1])` after 60s. Reproduced 3 times over 8 days on Mac16,8 / M4 Pro with identical signature in `/Library/Logs/DiagnosticReports/panic-full-*.panic`: panic(cpu N): busy timeout[1], (60s): 'IOMediaBSDClient' (1,1812001) @IOService.cpp:5986 Panicked task ... pid : watchdogd last started kext: com.apple.iokit.SCSITaskUserClient The existing `deadtimeout=45` option supports Finder's manual eject path but does not cover scheduled background I/O (Spotlight reindex, Time Machine attempts, mds_stores, daemon polling) that hits the dead mount after hot-unplug. macOS does not auto-teardown NFS mounts on physical disconnect — `DiskArbitration` only fires callbacks for registered listeners, which we don't have outside synchronous CLI flow (see fsutil.rs comment near line 206 acknowledging the gap). Soft-mount semantics with bounded timeouts return EIO after ~30s (3 retries × 10s `timeo`) instead of holding the registry busy. Returning EIO is appropriate when the physical device is gone — operations that would have hung forever now produce a meaningful error and the kernel releases the IOKit entry. Includes regression test in `fsutil::tests::default_nfs_opts_include_soft_mount_semantics`. Discussed in GitHub issue (to be filed alongside this PR). --- anylinuxfs/src/fsutil.rs | 65 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/anylinuxfs/src/fsutil.rs b/anylinuxfs/src/fsutil.rs index 5af6918..67bb51a 100644 --- a/anylinuxfs/src/fsutil.rs +++ b/anylinuxfs/src/fsutil.rs @@ -112,6 +112,24 @@ impl Default for NfsOptions { { opts.insert("deadtimeout".into(), "45".into()); // this is what Finder uses opts.insert("nfc".into(), "".into()); // NFC Unicode normalization (macOS-only) + + // Soft mount semantics to bound kernel-level retries when the + // underlying microVM becomes unreachable (e.g. user hot-unplugs + // a managed USB drive without running `anylinuxfs unmount` first). + // + // Without this, macOS NFS client (default hard mount) retries + // indefinitely against the dead NFS server, holds IOMediaBSDClient + // busy, and triggers `panic(busy timeout[1])` once kernel watchdogd + // notices a registry entry stuck for 60s. + // + // `deadtimeout=45` above helps Finder's manual eject path but does + // not cover scheduled background I/O (Spotlight, Time Machine, + // mds_stores) that hits the dead mount after hot-unplug. The + // combination of `soft,timeo=100,retrans=3` returns EIO after + // ~30s, which is appropriate when the VM/disk is gone. + opts.insert("soft".into(), "".into()); + opts.insert("timeo".into(), "100".into()); // tenths of a second → 10s per try + opts.insert("retrans".into(), "3".into()); } opts.insert(NOLOCK_KEY.into(), "".into()); opts.insert("vers".into(), "3".into()); @@ -443,3 +461,50 @@ pub fn wait_for_file(file: impl AsRef) -> anyhow::Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Regression test: default macOS NFS opts must include soft-mount semantics + /// (`soft`, `timeo=100`, `retrans=3`) so the macOS NFS client bounds its + /// retries when the underlying microVM becomes unreachable (e.g. user + /// hot-unplugs a managed USB drive without running `anylinuxfs unmount`). + /// + /// Without these options, the kernel NFS client retries indefinitely and + /// holds `IOMediaBSDClient` busy until `watchdogd` triggers a kernel panic + /// after 60s (`panic(busy timeout[1])`). See discussion in PR adding these + /// options for the original incident reports. + #[test] + #[cfg(target_os = "macos")] + fn default_nfs_opts_include_soft_mount_semantics() { + let opts = NfsOptions::default(); + let opts_str = String::from_utf8(opts.to_list()) + .expect("NfsOptions::to_list() should produce valid UTF-8 for ASCII keys"); + + assert!( + opts_str.contains("soft"), + "missing 'soft' option in default macOS NFS opts: {opts_str}" + ); + assert!( + opts_str.contains("timeo=100"), + "missing 'timeo=100' option (per-retry timeout): {opts_str}" + ); + assert!( + opts_str.contains("retrans=3"), + "missing 'retrans=3' option (max retransmissions): {opts_str}" + ); + + // Existing defaults must remain — these support Finder eject path and + // mount stability. Soft-mount additions are complementary, not a + // replacement. + assert!( + opts_str.contains("deadtimeout=45"), + "existing 'deadtimeout=45' removed: {opts_str}" + ); + assert!( + opts_str.contains("vers=3"), + "existing 'vers=3' removed: {opts_str}" + ); + } +}