Create sandboxes directly from snapshots (#1459)

* Introduce HostFunctions newtype for sandbox construction

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Expose SandboxMemoryLayout fields to crate

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Capture host function metadata in Snapshot

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Make gdb work for already initialised snapshots

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Add MultiUseSandbox from_snapshot

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Add gdb test for MultiUseSandbox from_snapshot

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

* Add tests for MultiUseSandbox from_snapshot

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

---------

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

Author: ludfjig

src/hyperlight_host/examples/guest-debugging/main.rs

@@ -115,6 +115,76 @@ mod tests {
     #[cfg(windows)]
     const GDB_COMMAND: &str = "gdb";
 
+    /// Construct the (out_file_path, cmd_file_path, manifest_dir)
+    /// triple every gdb test needs.
+    fn gdb_test_paths(name: &str) -> (String, String, String) {
+        let out_dir = std::env::var("OUT_DIR").expect("Failed to get out dir");
+        let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
+            .expect("Failed to get manifest dir")
+            .replace('\\', "/");
+        let out_file_path = format!("{out_dir}/{name}.output");
+        let cmd_file_path = format!("{out_dir}/{name}-commands.txt");
+        (out_file_path, cmd_file_path, manifest_dir)
+    }
+
+    /// Build a gdb script that connects to `port`, sets a single
+    /// breakpoint at `breakpoint`, prints `echo_msg` when hit, and
+    /// detaches before quitting.
+    ///
+    /// The breakpoint commands end with `detach` + `quit` instead of
+    /// `continue`. The previous "inner continue, outer continue, quit"
+    /// shape races with the inferior exit. After the breakpoint hits
+    /// and the inner `continue` resumes the guest, the guest may run
+    /// to completion and the gdb stub may close the remote before gdb
+    /// has dispatched the outer `continue`, producing a non-zero exit
+    /// with `Remote connection closed`. Detaching from the breakpoint
+    /// commands removes that window. The host process keeps running
+    /// the guest call to completion on its own after detach.
+    fn single_breakpoint_script(
+        manifest_dir: &str,
+        port: u16,
+        out_file_path: &str,
+        breakpoint: &str,
+        echo_msg: &str,
+    ) -> String {
+        let cmd = format!(
+            "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
+                target remote :{port}
+
+                set pagination off
+                set logging file {out_file_path}
+                set logging enabled on
+
+                break {breakpoint}
+                    commands
+                    echo \"{echo_msg}\\n\"
+                    backtrace
+
+                    set logging enabled off
+                    detach
+                    quit
+                end
+
+                continue
+            "
+        );
+        #[cfg(windows)]
+        let cmd = format!("set osabi none\n{cmd}");
+        cmd
+    }
+
+    /// Spawn the gdb client to execute the script in `cmd_file_path`.
+    fn spawn_gdb_client(cmd_file_path: &str) -> std::process::Child {
+        Command::new(GDB_COMMAND)
+            .arg("-nx")
+            .arg("--nw")
+            .arg("--batch")
+            .arg("-x")
+            .arg(cmd_file_path)
+            .spawn()
+            .expect("Failed to start gdb")
+    }
+
     fn write_cmds_file(cmd_file_path: &str, cmd: &str) -> io::Result<()> {
         let file = File::create(cmd_file_path)?;
         let mut writer = BufWriter::new(file);
@@ -163,14 +233,7 @@ mod tests {
         // wait 3 seconds for the gdb to connect
         thread::sleep(Duration::from_secs(3));
 
-        let mut gdb = Command::new(GDB_COMMAND)
-            .arg("-nx") // Don't load any .gdbinit files
-            .arg("--nw")
-            .arg("--batch")
-            .arg("-x")
-            .arg(cmd_file_path)
-            .spawn()
-            .map_err(|e| new_error!("Failed to start gdb process: {}", e))?;
+        let mut gdb = spawn_gdb_client(cmd_file_path);
 
         // wait 3 seconds for the gdb to connect
         thread::sleep(Duration::from_secs(10));
@@ -245,38 +308,16 @@ mod tests {
     #[test]
     #[serial]
     fn test_gdb_end_to_end() {
-        let out_dir = std::env::var("OUT_DIR").expect("Failed to get out dir");
-        let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
-            .expect("Failed to get manifest dir")
-            .replace('\\', "/");
-        let out_file_path = format!("{out_dir}/gdb.output");
-        let cmd_file_path = format!("{out_dir}/gdb-commands.txt");
-
-        let cmd = format!(
-            "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
-                target remote :8080
-
-                set pagination off
-                set logging file {out_file_path}
-                set logging enabled on
-
-                break hyperlight_main
-                    commands
-                    echo \"Stopped at hyperlight_main breakpoint\\n\"
-                    backtrace
-
-                    set logging enabled off
-                    detach
-                    quit
-                end
-
-                continue
-            "
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb");
+
+        let cmd = single_breakpoint_script(
+            &manifest_dir,
+            8080,
+            &out_file_path,
+            "hyperlight_main",
+            "Stopped at hyperlight_main breakpoint",
         );
 
-        #[cfg(windows)]
-        let cmd = format!("set osabi none\n{}", cmd);
-
         let checker = |contents: String| contents.contains("Stopped at hyperlight_main breakpoint");
 
         let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
@@ -288,13 +329,8 @@ mod tests {
     #[test]
     #[serial]
     fn test_gdb_sse_check() {
-        let out_dir = std::env::var("OUT_DIR").expect("Failed to get out dir");
-        let manifest_dir = std::env::var("CARGO_MANIFEST_DIR")
-            .expect("Failed to get manifest dir")
-            .replace('\\', "/");
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-sse");
         println!("manifest dir {manifest_dir}");
-        let out_file_path = format!("{out_dir}/gdb-sse.output");
-        let cmd_file_path = format!("{out_dir}/gdb-sse--commands.txt");
 
         let cmd = format!(
             "file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
@@ -330,4 +366,74 @@ mod tests {
         cleanup(&out_file_path, &cmd_file_path);
         assert!(result.is_ok(), "{}", result.unwrap_err());
     }
+
+    #[test]
+    #[serial]
+    fn test_gdb_from_snapshot() {
+        use hyperlight_host::HostFunctions;
+
+        const PORT: u16 = 8081;
+
+        let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-from-snapshot");
+
+        // Build a sandbox the normal way and snapshot it in-memory.
+        let mut producer: MultiUseSandbox = UninitializedSandbox::new(
+            hyperlight_host::GuestBinary::FilePath(
+                hyperlight_testing::simple_guest_as_string().unwrap(),
+            ),
+            None,
+        )
+        .unwrap()
+        .evolve()
+        .unwrap();
+        let snap = producer.snapshot().unwrap();
+
+        // Order matters. The gdb stub event loop must enter (i.e.
+        // `VcpuStopped` must be sent on the channel) before the gdb
+        // client connects, otherwise the wire protocol desyncs. The
+        // evolve case gets this for free because `evolve()` runs
+        // `vm.initialise()` which trips the entry breakpoint
+        // immediately. For a `Call` snapshot `vm.initialise` is a
+        // no-op, so we trigger the breakpoint by running `sbox.call`
+        // here before the client is launched below.
+        let snap_thread = snap.clone();
+        let sandbox_thread = thread::spawn(move || -> Result<()> {
+            let mut cfg = SandboxConfiguration::default();
+            cfg.set_guest_debug_info(DebugInfo { port: PORT });
+
+            let mut sbox =
+                MultiUseSandbox::from_snapshot(snap_thread, HostFunctions::default(), Some(cfg))?;
+            sbox.call::<i32>(
+                "PrintOutput",
+                "Hello from a from_snapshot sandbox\n".to_string(),
+            )?;
+            Ok(())
+        });
+
+        // Wait for the sandbox thread to bind the listener, install
+        // the one-shot breakpoint, and trip it.
+        thread::sleep(Duration::from_secs(3));
+
+        let cmd = single_breakpoint_script(
+            &manifest_dir,
+            PORT,
+            &out_file_path,
+            "main.rs:simpleguest::print_output",
+            "Stopped at print_output breakpoint",
+        );
+        write_cmds_file(&cmd_file_path, &cmd).expect("Failed to write gdb commands");
+
+        let mut gdb = spawn_gdb_client(&cmd_file_path);
+        let _ = gdb.wait();
+        let sandbox_result = sandbox_thread
+            .join()
+            .expect("from_snapshot sandbox thread panicked");
+
+        let checker = |contents: String| contents.contains("Stopped at print_output breakpoint");
+        let result = check_output(&out_file_path, checker);
+
+        cleanup(&out_file_path, &cmd_file_path);
+        sandbox_result.expect("from_snapshot sandbox returned error");
+        result.expect("gdb output missing expected breakpoint hit");
+    }
 }

src/hyperlight_host/src/func/host_functions.rs

@@ -52,7 +52,8 @@ impl Registerable for UninitializedSandbox {
             return_type: Output::TYPE,
         };
 
-        (*hfs).register_host_function(name.to_string(), entry)
+        (*hfs).register_host_function(name.to_string(), entry);
+        Ok(())
     }
 }
 
@@ -92,7 +93,31 @@ impl Registerable for crate::MultiUseSandbox {
             return_type: Output::TYPE,
         };
 
-        (*hfs).register_host_function(name.to_string(), entry)
+        (*hfs).register_host_function(name.to_string(), entry);
+
+        // Registration mutates the host-function set captured in
+        // snapshots. Invalidate the cached snapshot so the next
+        // `snapshot()` call reflects the updated registry.
+        self.snapshot = None;
+        Ok(())
+    }
+}
+
+impl Registerable for crate::HostFunctions {
+    fn register_host_function<Args: ParameterTuple, Output: SupportedReturnType>(
+        &mut self,
+        name: &str,
+        hf: impl Into<HostFunction<Output, Args>>,
+    ) -> Result<()> {
+        let entry = FunctionEntry {
+            function: hf.into().into(),
+            parameter_types: Args::TYPE,
+            return_type: Output::TYPE,
+        };
+
+        self.inner_mut()
+            .register_host_function(name.to_string(), entry);
+        Ok(())
     }
 }
 
@@ -236,7 +261,7 @@ pub(crate) fn register_host_function<Args: ParameterTuple, Output: SupportedRetu
         .host_funcs
         .try_lock()
         .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
-        .register_host_function(name.to_string(), entry)?;
+        .register_host_function(name.to_string(), entry);
 
     Ok(())
 }

src/hyperlight_host/src/hypervisor/gdb/arch.rs

@@ -61,9 +61,8 @@ pub(crate) const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;
 /// Determine the reason the vCPU stopped
 /// This is done by checking the DR6 register and the exception id
 pub(crate) fn vcpu_stop_reason(
-    vm: &mut dyn DebuggableVm,
+    vm: &dyn DebuggableVm,
     dr6: u64,
-    entrypoint: u64,
     exception: u32,
 ) -> std::result::Result<VcpuStopReason, VcpuStopReasonError> {
     let CommonRegisters { rip, .. } = vm.regs()?;
@@ -81,10 +80,6 @@ pub(crate) fn vcpu_stop_reason(
         // Check page 19-4 Vol. 3B of Intel 64 and IA-32
         // Architectures Software Developer's Manual
         if DR6_HW_BP_FLAGS_MASK & dr6 != 0 {
-            if rip == entrypoint {
-                vm.remove_hw_breakpoint(entrypoint)?;
-                return Ok(VcpuStopReason::EntryPointBp);
-            }
             return Ok(VcpuStopReason::HwBp);
         }
     }
@@ -98,12 +93,10 @@ pub(crate) fn vcpu_stop_reason(
         r"The vCPU exited because of an unknown reason:
         rip: {:?}
         dr6: {:?}
-        entrypoint: {:?}
         exception: {:?}
         ",
         rip,
         dr6,
-        entrypoint,
         exception,
     );
 

src/hyperlight_host/src/hypervisor/gdb/event_loop.rs

@@ -59,7 +59,6 @@ impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop {
                     // Resume execution if unknown reason for stop
                     let stop_response = match stop_reason {
                         VcpuStopReason::DoneStep => BaseStopReason::DoneStep,
-                        VcpuStopReason::EntryPointBp => BaseStopReason::HwBreak(()),
                         VcpuStopReason::SwBp => BaseStopReason::SwBreak(()),
                         VcpuStopReason::HwBp => BaseStopReason::HwBreak(()),
                         // This is a consequence of the GDB client sending an interrupt signal

src/hyperlight_host/src/hypervisor/gdb/mod.rs

@@ -171,10 +171,6 @@ impl DebugMemoryAccess {
 pub enum VcpuStopReason {
     Crash,
     DoneStep,
-    /// Hardware breakpoint inserted by the hypervisor so the guest can be stopped
-    /// at the entry point. This is used to avoid the guest from executing
-    /// the entry point code before the debugger is connected
-    EntryPointBp,
     HwBp,
     SwBp,
     Interrupt,

src/hyperlight_host/src/hypervisor/hyperlight_vm/mod.rs

@@ -398,6 +398,13 @@ pub(crate) struct HyperlightVm {
     pub(super) gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(gdb)]
     pub(super) sw_breakpoints: HashMap<u64, u8>, // addr -> original instruction
+    /// One-shot hw breakpoint installed at the entry address when gdb is
+    /// enabled, so the gdb stub gets a `VcpuStopped` to enter its event
+    /// loop on the first vCPU run after construction. Cleared by the
+    /// `VmExit::Debug` arm of `run` the first time a `HwBp` stop fires
+    /// at the entry address.
+    #[cfg(gdb)]
+    pub(super) one_shot_entry_bp: Option<u64>,
     #[cfg(feature = "mem_profile")]
     pub(super) trace_info: MemTraceInfo,
     #[cfg(crashdump)]
@@ -654,17 +661,28 @@ impl HyperlightVm {
             match exit_reason {
                 #[cfg(gdb)]
                 Ok(VmExit::Debug { dr6, exception }) => {
-                    let initialise = match self.entrypoint {
-                        NextAction::Initialise(initialise) => initialise,
-                        _ => 0,
-                    };
-                    // Handle debug event (breakpoints)
+                    // Classify the debug exit. `vcpu_stop_reason` is a
+                    // pure classifier and has no side effects on the VM.
                     let stop_reason = crate::hypervisor::gdb::arch::vcpu_stop_reason(
-                        self.vm.as_mut(),
+                        self.vm.as_ref(),
                         dr6,
-                        initialise,
                         exception,
                     )?;
+                    // Remove the one-shot entry breakpoint installed by
+                    // `HyperlightVm::new` the first time it fires so it
+                    // does not interfere with later user-installed
+                    // breakpoints at the same address.
+                    if matches!(stop_reason, VcpuStopReason::HwBp)
+                        && let Some(entry_addr) = self.one_shot_entry_bp
+                    {
+                        let rip = self.vm.regs().map_err(VcpuStopReasonError::GetRegs)?.rip;
+                        if rip == entry_addr {
+                            self.vm
+                                .remove_hw_breakpoint(entry_addr)
+                                .map_err(VcpuStopReasonError::RemoveHwBreakpoint)?;
+                            self.one_shot_entry_bp = None;
+                        }
+                    }
                     if let Err(e) = self.handle_debug(dbg_mem_access_fn.clone(), stop_reason) {
                         break Err(e.into());
                     }