Add crashdump example and include snapshot/scratch in core dumps (#1264)

* Include snapshot and scratch regions in crash dumps

Co-authored-by: Doru Blânzeanu <dblnz@pm.me>
Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* Add sentinel page for clean GDB backtraces

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* Tighten runtime config plumbing for crash dumps

The runtime config was passed by reference into set_up_hypervisor_partition then
immediately cloned, but no caller needs it afterward so it is now passed by
value. The entry point field uses Option<u64> instead of a bare zero default so
a missing value is detectable rather than silently producing a bogus AT_ENTRY.

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* Add explicit dump directory API for crash dumps

generate_crashdump_to_dir accepts the output directory
as a parameter instead of requiring callers to set the
HYPERLIGHT_CORE_DUMP_DIR environment variable. This
removes the need for unsafe std::env::set_var in tests
while preserving the existing env-var fallback path
for the automatic dump and the no-argument generate_crashdump method.

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* fixup! Include snapshot and scratch regions in crash dumps

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* fixup! Add sentinel page for clean GDB backtraces

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* fixup! Include snapshot and scratch regions in crash dumps

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

* fixup! Add sentinel page for clean GDB backtraces

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>

---------

Signed-off-by: James Sturtevant <jsturtevant@gmail.com>
Co-authored-by: Doru Blânzeanu <dblnz@pm.me>

Author: jsturtevant

Justfile

@@ -248,6 +248,7 @@ test-rust-gdb-debugging target=default-target features="":
 # rust test for crashdump
 test-rust-crashdump target=default-target features="":
     {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }} -- test_crashdump
+    {{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }} --example crashdump {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }}
 
 # rust test for tracing
 test-rust-tracing target=default-target features="":
@@ -353,6 +354,7 @@ run-rust-examples target=default-target features="":
 run-rust-examples-linux target=default-target features="": (run-rust-examples target features)
     {{ cargo-cmd }} run --profile={{ if target == "debug" { "dev" } else { target } }}   {{ target-triple-flag }} --example tracing {{ if features =="" {''} else { "--features " + features } }}
     {{ cargo-cmd }} run --profile={{ if target == "debug" { "dev" } else { target } }}   {{ target-triple-flag }}  --example tracing {{ if features =="" {"--features function_call_metrics" } else {"--features function_call_metrics," + features} }}
+    {{ cargo-cmd }} run --profile={{ if target == "debug" { "dev" } else { target } }}   {{ target-triple-flag }} --example crashdump {{ if features =="" {'--features crashdump'} else { "--features crashdump," + features } }}
 
 
 #########################

docs/how-to-debug-a-hyperlight-guest.md

@@ -264,6 +264,48 @@ The crashdump should be available `/tmp` or in the crash dump directory (see `HY
 After the core dump has been created, to inspect the state of the guest, load the core dump file using `gdb` or `lldb`.
 **NOTE: This feature has been tested with version `15.0` of `gdb` and version `17` of `lldb`, earlier versions may not work, it is recommended to use these versions or later.**
 
+#### Using gdb
+
+Load the core dump alongside the guest binary that was running when the crash occurred:
+
+```bash
+gdb <guest-binary> -c <core-dump-file>
+```
+
+For example:
+
+```bash
+gdb src/tests/rust_guests/bin/debug/simpleguest -c /tmp/hl_dumps/hl_core_20260225_T165358.517.elf
+```
+
+Common commands for inspecting the dump:
+
+```gdb
+# View all general-purpose registers (rip, rsp, rflags, etc.)
+(gdb) info registers
+
+# Disassemble around the crash site
+(gdb) x/10i $rip
+
+# View the stack
+(gdb) x/16xg $rsp
+
+# Backtrace (requires debug info in guest binary)
+(gdb) bt
+
+# List all memory regions in the dump (snapshot, scratch, mapped regions)
+(gdb) info files
+
+# Read memory at a specific address
+(gdb) x/s <address>        # null-terminated string
+(gdb) x/32xb <address>     # 32 bytes in hex
+```
+
+See the `crashdump` example (`cargo run --example crashdump --features crashdump`)
+for a runnable demonstration of both automatic and on-demand crash dumps.
+
+#### Using VSCode
+
 To do this in vscode, the following configuration can be used to add debug configurations:
 
 ```vscode

src/hyperlight_guest_bin/src/arch/amd64/dispatch.rs

@@ -58,6 +58,8 @@ unsafe extern "C" {
 core::arch::global_asm!("
     .global dispatch_function
     dispatch_function:
+    .cfi_startproc
+    .cfi_undefined rip
     jnz flush_done
     mov rdi, cr4
     xor rdi, 0x80
@@ -67,4 +69,5 @@ core::arch::global_asm!("
     flush_done:
     call {internal_dispatch_function}\n
     hlt\n
+    .cfi_endproc
 ", internal_dispatch_function = sym crate::guest_function::call::internal_dispatch_function);

src/hyperlight_guest_bin/src/arch/amd64/init.rs

@@ -157,10 +157,22 @@ unsafe extern "C" {
     ) -> !;
 }
 
+// Mark this as the bottom-most frame for debuggers:
+//  - .cfi_undefined rip: tells DWARF unwinders there is no return
+//    address to recover (i.e. no caller frame).
+//  - xor ebp, ebp: sets the frame pointer to zero so frame-pointer-
+//    based unwinders recognise this as the end of the chain.
+// See System V AMD64 ABI: https://gitlab.com/x86-psABIs/x86-64-ABI
+//      §3.4.1 (Initial Stack and Register State)
+//      §6.3 Unwinding Through Assembler Code
 core::arch::global_asm!("
     .global pivot_stack\n
     pivot_stack:\n
+    .cfi_startproc\n
+    .cfi_undefined rip\n
     mov rsp, r8\n
+    xor ebp, ebp\n
     call {generic_init}\n
     hlt\n
+    .cfi_endproc\n
 ", generic_init = sym crate::generic_init);