feat: add mcall support and fix systemstack arm64 register order

Author: wehzzz

interpreter/golabels/runtime_data.go

@@ -27,21 +27,27 @@ func getOffsets(vers string) support.GoLabelsOffsets {
 		Hmap_log2_bucket_count: 0,
 		// https://github.com/golang/go/blob/6885bad7dd86880be6929c0/src/runtime/map.go#L118
 		Hmap_buckets: 0,
-		// Absolute offsets from g struct start for g.sched (gobuf) fields.
-		// g.sched starts at offset 56 (immediately after g.m at offset 48, pointer 8 bytes).
-		// gobuf.sp is at offset 0 within gobuf - stable across all supported Go versions.
-		// https://github.com/golang/go/blob/80e2e474b8d9124d03b744f/src/runtime/runtime2.go#L325
+		// g.sched is a gobuf struct immediately following g.m (offset 48 + 8 = 56).
+		// gobuf.sp is the first field (offset 0 in gobuf), gobuf.pc is the second (offset 8).
+		// https://github.com/golang/go/blob/80e2e474b8d9124d03b744f4e2da099a4eec5957/src/runtime/runtime2.go#L311
 		Sched_sp: 56,
+		Sched_pc: 64,
+		// gobuf.bp is at offset 48 within gobuf in go1.24 and earlier. In go1.25 and later,
+		// it is at offset 40 because of ret field removal. (offset 56 + 48 = 104)
+		// go1.25: https://github.com/golang/go/blob/6e676ab2b809d46623acb5988248d95d1eb7939c/src/runtime/runtime2.go#L315
+		Sched_bp: 104,
 	}
 
 	// Version enforcement takes place in the Loader function.
 	if version.Compare(vers, "go1.26") >= 0 {
 		offsets.Curg = 184
 		offsets.Labels = 352
+		offsets.Sched_bp = 96
 		return offsets
 	} else if version.Compare(vers, "go1.25") >= 0 {
 		offsets.Curg = 184
 		offsets.Labels = 344
+		offsets.Sched_bp = 96
 		return offsets
 	} else if version.Compare(vers, "go1.24") >= 0 {
 		offsets.Labels = 352

nativeunwind/elfunwindinfo/elfgopclntab.go

@@ -32,10 +32,10 @@ var goFunctionsStopDelta = map[string]*sdtypes.UnwindInfo{
 	"runtime.goexit": &sdtypes.UnwindInfoStop, // return address in all goroutine stacks
 
 	// Stack switch functions: when encountered during unwinding on the g0 (system) stack,
-	// the unwinder crosses back to the goroutine stack by reading the goroutine's saved
-	// context from gobuf.
+	// the unwinder crosses back to the goroutine stack using the goroutine's saved
+	// context from g.sched (gobuf).
 	"runtime.systemstack": &sdtypes.UnwindInfoGoSystemstack,
-	"runtime.mcall":       &sdtypes.UnwindInfoStop,
+	"runtime.mcall":       &sdtypes.UnwindInfoGoMcall,
 
 	// signal return frame
 	"runtime.sigreturn":            &sdtypes.UnwindInfoSignal,

nativeunwind/stackdeltatypes/stackdeltatypes.go

@@ -49,10 +49,18 @@ var UnwindInfoFramePointer = UnwindInfo{Flags: support.UnwindFlagCommand,
 // UnwindInfoGoSystemstack is the stack delta info for runtime.systemstack.
 // When encountered during unwinding, the unwinder crosses from the g0 system
 // stack to the goroutine stack by reading the saved FP and RA from the frame
-// pointer prologue at gobuf.sp (which equals gobuf.bp on both amd64 and arm64).
+// pointer prologue at gobuf.sp.
 var UnwindInfoGoSystemstack = UnwindInfo{Flags: support.UnwindFlagCommand,
 	Param: support.UnwindCommandGoSystemstack}
 
+// UnwindInfoGoMcall is the stack delta info for runtime.mcall.
+// When encountered during unwinding, the unwinder crosses from the g0 system
+// stack to the goroutine stack by reading the caller's saved registers directly
+// from gobuf.{pc, sp, bp}. If m.curg is nil (after dropg), the goroutine pointer
+// is recovered from the g0 stack at *(g0.sched.sp - 8).
+var UnwindInfoGoMcall = UnwindInfo{Flags: support.UnwindFlagCommand,
+	Param: support.UnwindCommandGoMcall}
+
 // UnwindInfoLR contains the description to unwind ARM64 function without a frame (LR only)
 var UnwindInfoLR = UnwindInfo{
 	BaseReg:    support.UnwindRegSp,

processmanager/execinfomanager/manager.go

@@ -127,9 +127,9 @@ func NewExecutableInfoManager(
 
 	interpreterLoaders = append(interpreterLoaders, apmint.Loader)
 
-	// Always register golabels loader: the native unwinder needs Go runtime
+	// Register golabels loader. The native unwinder needs Go runtime
 	// offsets (m, curg, g.sched) to cross the systemstack boundary,
-	// regardless of whether custom labels extraction is enabled.
+	// to perform Go stack unwinding.
 	interpreterLoaders = append(interpreterLoaders, golabels.Loader)
 
 	deferredFileIDs, err := lru.NewSynced[host.FileID, libpf.Void](deferredFileIDSize,

support/ebpf/go_runtime.h

@@ -7,22 +7,23 @@
 #include "tsd.h"
 #include "types.h"
 
-// get_m_ptr reads the M (machine/OS thread) pointer for the current goroutine.
-// It does so by reading the G (goroutine) pointer from thread-local storage,
-// then following the g.m pointer.
+// get_g_ptr reads the current G (goroutine) pointer from thread-local storage.
+// TLS always contains the G that is currently executing on the thread. During
+// systemstack/mcall, this is g0 (the system goroutine) since we are on the
+// system stack.
 //
 // On aarch64, when tls_offset is 0 (non-CGO binaries), the G pointer is taken
 // from the r28 register saved in the unwind state instead of TLS.
-static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *state)
+static EBPF_INLINE u64 get_g_ptr(struct GoLabelsOffsets *offs, UnwindState *state)
 {
 #if defined(__x86_64__)
   (void)state;
 #endif
   u64 g_addr     = 0;
   void *tls_base = NULL;
   if (tsd_get_base(&tls_base) < 0) {
-    DEBUG_PRINT("cl: failed to get tsd base; can't read m_ptr");
-    return NULL;
+    DEBUG_PRINT("cl: failed to get tsd base; can't read g_addr");
+    return 0;
   }
   DEBUG_PRINT(
     "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset);
@@ -33,17 +34,31 @@ static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *st
     g_addr = state->r28;
 #elif defined(__x86_64__)
     DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64");
-    return NULL;
+    return 0;
 #endif
   }
 
   if (g_addr == 0) {
     if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) {
       DEBUG_PRINT("cl: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
-      return NULL;
+      return 0;
     }
   }
 
+  DEBUG_PRINT("cl: g_addr 0x%lx", (unsigned long)g_addr);
+  return g_addr;
+}
+
+// get_m_ptr reads the M (machine/OS thread) pointer for the current goroutine.
+// It does so by reading the G (goroutine) pointer from thread-local storage,
+// then following the g.m pointer.
+static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *state)
+{
+  u64 g_addr = get_g_ptr(offs, state);
+  if (!g_addr) {
+    return NULL;
+  }
+
   DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset);
   void *m_ptr_addr;
   if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) {
@@ -54,4 +69,4 @@ static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *st
   return m_ptr_addr;
 }
 
-#endif // OPTI_GO_RUNTIME_H
+#endif // OPTI_GO_RUNTIME_H