feat: add ERR_REENTRANT deadlock detection for host callbacks

When a host function or print callback attempts to call sandbox methods
(callHandler, snapshot, restore, unload) while guest code is executing,
return ERR_REENTRANT immediately instead of deadlocking.

Implementation:
- Add executing_flag (AtomicBool) to LoadedJSSandboxWrapper
- Set flag true during spawn_blocking, false after completion
- with_inner/take_inner_with use try_lock when flag is set
- New ErrorCode::Reentrant variant with ERR_REENTRANT code

This primarily prevents guaranteed deadlocks in host function callbacks
(where the Rust side block_on's waiting for the result while holding
the lock). For print callbacks (fire-and-forget) it's a safety net.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: simongdavies

src/js-host-api/src/lib.rs

@@ -123,6 +123,8 @@ enum ErrorCode {
     Consumed,
     /// Internal / unexpected failure (lock poison, task join error, etc.).
     Internal,
+    /// Reentrant call detected — calling sandbox methods from a host callback.
+    Reentrant,
 }
 
 impl ErrorCode {
@@ -135,6 +137,7 @@ impl ErrorCode {
             Self::InvalidArg => "ERR_INVALID_ARG",
             Self::Consumed => "ERR_CONSUMED",
             Self::Internal => "ERR_INTERNAL",
+            Self::Reentrant => "ERR_REENTRANT",
         }
     }
 }
@@ -230,6 +233,15 @@ fn join_error(err: tokio::task::JoinError) -> napi::Error {
     )
 }
 
+/// Creates an error for reentrant calls detected at runtime.
+fn reentrant_error() -> napi::Error {
+    hl_error(
+        ErrorCode::Reentrant,
+        "Cannot call sandbox methods from a host callback — the sandbox lock is held during \
+         guest execution. Perform this operation after callHandler() resolves instead",
+    )
+}
+
 // ── Snapshot ─────────────────────────────────────────────────────────
 
 /// A captured point-in-time state of a sandbox.
@@ -435,9 +447,10 @@ impl SandboxBuilderWrapper {
             //
             // **Reentrancy note:** The print callback runs while the sandbox
             // Mutex is held (inside `call_handler`'s `spawn_blocking`).
-            // Calling Hyperlight APIs that acquire the same lock from within
-            // the callback (e.g. `snapshot()`, `restore()`, `unload()`) will
-            // deadlock. Keep print callbacks simple — logging only.
+            // If user code in the callback attempts to call methods that
+            // acquire the same lock (e.g. `snapshot()`, `restore()`,
+            // `unload()`, `callHandler()`), the `executing_flag` deadlock
+            // detection will return `ERR_REENTRANT` instead of hanging.
             let print_fn = move |msg: String| -> i32 {
                 let status = callback.call(msg, ThreadsafeFunctionCallMode::Blocking);
                 if status == Status::Ok {
@@ -853,6 +866,7 @@ impl JSSandboxWrapper {
             poisoned_flag,
             last_call_stats: Arc::new(ArcSwapOption::empty()),
             disposed_flag: Arc::new(AtomicBool::new(false)),
+            executing_flag: Arc::new(AtomicBool::new(false)),
         })
     }
 
@@ -935,17 +949,37 @@ pub struct LoadedJSSandboxWrapper {
     /// `unload()`), for lock-free checks in sync getters that don't
     /// consult the inner Mutex.
     disposed_flag: Arc<AtomicBool>,
+
+    /// Tracks whether guest code is currently executing inside `call_handler`.
+    ///
+    /// Used for deadlock detection: if a host callback (print fn, host function)
+    /// tries to call a method that needs the inner Mutex while this is `true`,
+    /// we return `ERR_REENTRANT` immediately instead of deadlocking.
+    executing_flag: Arc<AtomicBool>,
 }
 
 type LoadedJSSandboxGuard = OwnedMappedMutexGuard<Option<LoadedJSSandbox>, LoadedJSSandbox>;
 
 impl LoadedJSSandboxWrapper {
     /// Borrow the inner value mutably via Mutex, or error if consumed.
+    ///
+    /// Performs deadlock detection: if `executing_flag` is set (meaning we're
+    /// inside a `call_handler` on a background thread), `try_lock` is attempted
+    /// first. If it fails, we know this is a reentrant call from a host callback
+    /// and return `ERR_REENTRANT` immediately instead of deadlocking.
     async fn with_inner<R>(
         &self,
         f: impl AsyncFnOnce(LoadedJSSandboxGuard) -> napi::Result<R>,
     ) -> napi::Result<R> {
-        let sandbox = self.inner.clone().lock_owned().await;
+        let sandbox = if self.executing_flag.load(Ordering::Acquire) {
+            // We're inside a host callback — try_lock to detect reentrancy.
+            match self.inner.clone().try_lock_owned() {
+                Ok(guard) => guard,
+                Err(_) => return Err(reentrant_error()),
+            }
+        } else {
+            self.inner.clone().lock_owned().await
+        };
         let sandbox = OwnedMutexGuard::try_map(sandbox, Option::as_mut)
             .map_err(|_| consumed_error("LoadedJSSandbox"))?;
         f(sandbox).await
@@ -954,30 +988,42 @@ impl LoadedJSSandboxWrapper {
     /// Borrow the inner value mutably via Mutex, or error if consumed.
     /// The closure `f` will run using spawn_blocking, so it can perform long-running operations without
     /// blocking the Node.js event loop. This is the main way to interact with the inner `LoadedJSSandbox`.
+    ///
+    /// Sets `executing_flag` for the duration of the blocking closure so that
+    /// reentrant calls from host callbacks are detected instead of deadlocking.
     async fn with_blocking_inner<R: Send + 'static>(
         &self,
         f: impl FnOnce(LoadedJSSandboxGuard) -> napi::Result<R> + Send + 'static,
     ) -> napi::Result<R> {
+        let executing_flag = self.executing_flag.clone();
         self.with_inner(async move |sandbox| {
-            tokio::task::spawn_blocking(move || f(sandbox))
+            executing_flag.store(true, Ordering::Release);
+            let result = tokio::task::spawn_blocking(move || f(sandbox))
                 .await
-                .map_err(join_error)?
+                .map_err(join_error)?;
+            executing_flag.store(false, Ordering::Release);
+            result
         })
         .await
     }
 
     /// Take ownership of the inner value, returning a consumed-state error if
     /// this instance has already been used.
+    ///
+    /// Performs the same deadlock detection as `with_inner`.
     async fn take_inner_with<R>(
         &self,
         f: impl AsyncFnOnce(LoadedJSSandbox) -> napi::Result<R>,
     ) -> napi::Result<R> {
-        let sandbox = self
-            .inner
-            .lock()
-            .await
-            .take()
-            .ok_or_else(|| consumed_error("LoadedJSSandbox"))?;
+        let sandbox = if self.executing_flag.load(Ordering::Acquire) {
+            match self.inner.try_lock() {
+                Ok(mut guard) => guard.take(),
+                Err(_) => return Err(reentrant_error()),
+            }
+        } else {
+            self.inner.lock().await.take()
+        }
+        .ok_or_else(|| consumed_error("LoadedJSSandbox"))?;
         self.disposed_flag.store(true, Ordering::Release);
         f(sandbox).await
     }
@@ -986,14 +1032,20 @@ impl LoadedJSSandboxWrapper {
     /// this instance has already been used.
     /// The closure `f` will run using spawn_blocking, so it can perform long-running operations without
     /// blocking the Node.js event loop. This is the main way to interact with the inner `LoadedJSSandbox`.
+    ///
+    /// Sets `executing_flag` for the duration of the blocking closure.
     async fn take_blocking_inner_with<R: Send + 'static>(
         &self,
         f: impl FnOnce(LoadedJSSandbox) -> napi::Result<R> + Send + 'static,
     ) -> napi::Result<R> {
+        let executing_flag = self.executing_flag.clone();
         self.take_inner_with(async move |sandbox| {
-            tokio::task::spawn_blocking(move || f(sandbox))
+            executing_flag.store(true, Ordering::Release);
+            let result = tokio::task::spawn_blocking(move || f(sandbox))
                 .await
-                .map_err(join_error)?
+                .map_err(join_error)?;
+            executing_flag.store(false, Ordering::Release);
+            result
         })
         .await
     }