Enhance createdump to detect alt stack execution (#127068)

createdump's UnwindNativeFrames fails to capture the original thread
stack when a crash occurs on a thread using an alternate signal stack.
The native unwinder's monotonic-SP guard breaks we cross crosses the
signal trampoline back to the original stack, because the SP
legitimately decreases. This causes the unwinder to stop early, omitting
the original stack memory from the dump.

Use `unw_is_signal_frame` in `PAL_VirtualUnwindOutOfProc` to detect
signal trampoline frames. When a signal frame is detected,
`UnwindNativeFrames` allows a single SP decrease, enabling the unwinder
to cross back to the original stack and capture its memory.

Author: hoyosjs

src/coreclr/debug/createdump/createdumppal.cpp

@@ -23,7 +23,8 @@ typedef BOOL (*PFN_PAL_VirtualUnwindOutOfProc)(
     CONTEXT *context,
     PULONG64 functionStart,
     SIZE_T baseAddress,
-    UnwindReadMemoryCallback readMemoryCallback);
+    UnwindReadMemoryCallback readMemoryCallback,
+    bool *isSignalFrame);
 
 typedef BOOL (*PFN_PAL_GetUnwindInfoSize)(
     SIZE_T baseAddress,
@@ -129,13 +130,14 @@ PAL_VirtualUnwindOutOfProc(
     CONTEXT *context,
     PULONG64 functionStart,
     SIZE_T baseAddress,
-    UnwindReadMemoryCallback readMemoryCallback)
+    UnwindReadMemoryCallback readMemoryCallback,
+    bool *isSignalFrame)
 {
     if (!InitializePAL() || g_PAL_VirtualUnwindOutOfProc == nullptr)
     {
         return FALSE;
     }
-    return g_PAL_VirtualUnwindOutOfProc(context, functionStart, baseAddress, readMemoryCallback);
+    return g_PAL_VirtualUnwindOutOfProc(context, functionStart, baseAddress, readMemoryCallback, isSignalFrame);
 }
 
 BOOL

src/coreclr/debug/createdump/threadinfo.cpp

@@ -51,6 +51,8 @@ ThreadInfo::UnwindNativeFrames(CONTEXT* pContext)
     uint64_t previousSp = 0;
     uint64_t previousIp = 0;
     int ipMatchCount = 0;
+    bool isSignalFrame = false;
+    bool crossedSignalTrampoline = false;
 
     // For each native frame, add a page around the IP and any unwind info not already
     // added in VisitProgramHeader (Linux) and VisitSection (MacOS) to the dump.
@@ -69,10 +71,19 @@ ThreadInfo::UnwindNativeFrames(CONTEXT* pContext)
             sp++;
         }
 #endif
-        if (ip == 0 || sp <= previousSp) {
+        // When a signal handler uses SA_ONSTACK (alternate signal stack), the SP can legitimately
+        // decrease when unwinding crosses the signal trampoline back to the original thread stack.
+        // Allow the SP decrease if the current frame is a signal trampoline (detected by the
+        // previous unwind call) and we haven't already crossed one (limit to one crossing to
+        // bound corruption damage).
+        if (ip == 0 || sp == 0 || (sp <= previousSp && (!isSignalFrame || crossedSignalTrampoline))) {
             TRACE_VERBOSE("Unwind: sp not increasing or ip == 0 sp %p ip %p\n", (void*)sp, (void*)ip);
             break;
         }
+        if (sp < previousSp)
+        {
+            crossedSignalTrampoline = true;
+        }
         // Break out of the endless loop if the IP matches over a 1000 times. This is a fallback
         // behavior of libunwind when the module the IP is in doesn't have unwind info and for
         // simple stack overflows. The stack memory is added to the dump in GetThreadStack and
@@ -104,7 +115,8 @@ ThreadInfo::UnwindNativeFrames(CONTEXT* pContext)
 
         // Unwind the native frame adding all the memory accessed to the core dump via the read memory adapter.
         ULONG64 functionStart;
-        if (!PAL_VirtualUnwindOutOfProc(pContext, &functionStart, baseAddress, ReadMemoryAdapter)) {
+        isSignalFrame = false;
+        if (!PAL_VirtualUnwindOutOfProc(pContext, &functionStart, baseAddress, ReadMemoryAdapter, &isSignalFrame)) {
             TRACE("Unwind: PAL_VirtualUnwindOutOfProc returned false\n");
             break;
         }

src/coreclr/pal/inc/pal.h

@@ -2335,7 +2335,7 @@ typedef BOOL(*UnwindReadMemoryCallback)(PVOID address, PVOID buffer, SIZE_T size
 
 PALIMPORT BOOL PALAPI PAL_VirtualUnwind(CONTEXT *context);
 
-PALIMPORT BOOL PALAPI PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback);
+PALIMPORT BOOL PALAPI PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback, bool *isSignalFrame);
 
 PALIMPORT BOOL PALAPI PAL_GetUnwindInfoSize(SIZE_T baseAddress, ULONG64 ehFrameHdrAddr, UnwindReadMemoryCallback readMemoryCallback, PULONG64 ehFrameStart, PULONG64 ehFrameSize);
 

src/coreclr/pal/src/exception/remote-unwind.cpp

@@ -2328,17 +2328,23 @@ static unw_accessors_t unwind_accessors = init_unwind_accessors();
     functionStart - the pointer to return the starting address of the function or nullptr
     baseAddress - base address of the module to find the unwind info
     readMemoryCallback - reads memory from the target
+    isSignalFrame - output parameter: set to true if the unwound-to frame is a signal trampoline
 --*/
 BOOL
 PALAPI
-PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback)
+PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback, bool *isSignalFrame)
 {
     unw_addr_space_t addrSpace = 0;
     unw_cursor_t cursor;
     libunwindInfo info;
     BOOL result = FALSE;
     int st;
 
+    if (isSignalFrame)
+    {
+        *isSignalFrame = false;
+    }
+
     info.BaseAddress = baseAddress;
     info.Context = context;
     info.FunctionStart = 0;
@@ -2410,6 +2416,14 @@ PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T base
         goto exit;
     }
 
+    // Check if the frame we landed on is a signal trampoline. When a signal handler uses
+    // SA_ONSTACK, stepping from a signal frame crosses from the alternate signal stack to the
+    // original thread stack, which can cause the SP to decrease.
+    if (isSignalFrame && unw_is_signal_frame(&cursor) > 0)
+    {
+        *isSignalFrame = true;
+    }
+
     UnwindContextToContext(&cursor, context);
 
     result = TRUE;
@@ -2572,7 +2586,7 @@ PAL_GetUnwindInfoSize(SIZE_T baseAddress, ULONG64 ehFrameHdrAddr, UnwindReadMemo
 
 BOOL
 PALAPI
-PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback)
+PAL_VirtualUnwindOutOfProc(CONTEXT *context, PULONG64 functionStart, SIZE_T baseAddress, UnwindReadMemoryCallback readMemoryCallback, bool *isSignalFrame)
 {
     return FALSE;
 }