fix: go unwinding stops at systemstacks

interpreter/golabels/runtime_data.go

@@ -27,16 +27,29 @@ func getOffsets(vers string) support.GoLabelsOffsets {
 		Hmap_log2_bucket_count: 0,
 		// https://github.com/golang/go/blob/6885bad7dd86880be6929c0/src/runtime/map.go#L118
 		Hmap_buckets: 0,
+		// g.sched is a gobuf struct immediately following g.m (offset 48 + 8 = 56).
+		// gobuf.sp is the first field (offset 0 in gobuf).
+		// https://github.com/golang/go/blob/80e2e474b8d9124d03b744f4e2da099a4eec5957/src/runtime/runtime2.go#L311
+		Sched_sp: 56,
+		// Offsets of pc and bp within gobuf, relative to gobuf.sp.
+		// gobuf.pc is always at offset 8 (second field in gobuf).
+		Sched_pc_off: 8,
+		// gobuf.bp is at offset 48 within gobuf in go1.24 and earlier. In go1.25 and later,
+		// it is at offset 40 because of ret field removal.
+		// go1.25: https://github.com/golang/go/blob/6e676ab2b809d46623acb5988248d95d1eb7939c/src/runtime/runtime2.go#L315
+		Sched_bp_off: 48,
 	}
 
 	// Version enforcement takes place in the Loader function.
 	if version.Compare(vers, "go1.26") >= 0 {
 		offsets.Curg = 184
 		offsets.Labels = 352
+		offsets.Sched_bp_off = 40
 		return offsets
 	} else if version.Compare(vers, "go1.25") >= 0 {
 		offsets.Curg = 184
 		offsets.Labels = 344
+		offsets.Sched_bp_off = 40
 		return offsets
 	} else if version.Compare(vers, "go1.24") >= 0 {
 		offsets.Labels = 352

metrics/metrics.json

@@ -2096,5 +2096,47 @@
     "name": "UnwindErrBadDTVRead",
     "field": "bpf.errors.bad_dtv_read",
     "id": 286
+  },
+  {
+    "description": "Number of attempted Go mcall stack-switch unwinds",
+    "type": "counter",
+    "name": "UnwindGoMcallAttempts",
+    "field": "bpf.go.mcall_attempts",
+    "id": 287
+  },
+  {
+    "description": "Number of successful Go mcall stack-switch unwinds",
+    "type": "counter",
+    "name": "UnwindGoMcallSuccess",
+    "field": "bpf.go.mcall_success",
+    "id": 288
+  },
+  {
+    "description": "Number of Go mcall unwind failures due to missing Go offsets",
+    "type": "counter",
+    "name": "UnwindGoMcallErrNoGoOffsets",
+    "field": "bpf.go.errors.mcall_no_go_offsets",
+    "id": 289
+  },
+  {
+    "description": "Number of Go mcall unwind failures due to goroutine resolution",
+    "type": "counter",
+    "name": "UnwindGoMcallErrResolveGoroutine",
+    "field": "bpf.go.errors.mcall_resolve_goroutine",
+    "id": 290
+  },
+  {
+    "description": "Number of Go mcall unwind failures due to gobuf read errors",
+    "type": "counter",
+    "name": "UnwindGoMcallErrReadGobuf",
+    "field": "bpf.go.errors.mcall_read_gobuf",
+    "id": 291
+  },
+  {
+    "description": "Number of Go mcall unwind failures due to unpopulated gobuf",
+    "type": "counter",
+    "name": "UnwindGoMcallErrGobufNotPopulated",
+    "field": "bpf.go.errors.mcall_gobuf_not_populated",
+    "id": 292
   }
 ]

nativeunwind/elfunwindinfo/elfgopclntab.go

@@ -31,10 +31,12 @@ var goFunctionsStopDelta = map[string]*sdtypes.UnwindInfo{
 	"runtime.mstart": &sdtypes.UnwindInfoStop, // topmost for the go runtime main stacks
 	"runtime.goexit": &sdtypes.UnwindInfoStop, // return address in all goroutine stacks
 
-	// stack switch functions that would need special handling for further unwinding.
-	// See PF-1101.
-	"runtime.mcall":       &sdtypes.UnwindInfoStop,
-	"runtime.systemstack": &sdtypes.UnwindInfoStop,
+	// Stack switch functions: systemstack preserves the frame pointer chain across
+	// the g0/user stack boundary, so standard FP unwinding traverses it naturally.
+	// mcall clears BP/R29 before calling fn, breaking the FP chain - it needs a
+	// custom command that reads gobuf.{pc, sp, bp} directly.
+	"runtime.systemstack": &sdtypes.UnwindInfoFramePointer,
+	"runtime.mcall":       &sdtypes.UnwindInfoGoMcall,
 
 	// signal return frame
 	"runtime.sigreturn":            &sdtypes.UnwindInfoSignal,

nativeunwind/stackdeltatypes/stackdeltatypes.go

@@ -46,6 +46,14 @@ var UnwindInfoFramePointer = UnwindInfo{Flags: support.UnwindFlagCommand,
 	Param: support.UnwindCommandFramePointer,
 }
 
+// UnwindInfoGoMcall is the stack delta info for runtime.mcall.
+// When encountered during unwinding, the unwinder crosses from the g0 system
+// stack to the goroutine stack by reading the caller's saved registers directly
+// from gobuf.{pc, sp, bp}. If m.curg is nil (after dropg), the goroutine pointer
+// is recovered from the g0 stack at *(g0.sched.sp - 8).
+var UnwindInfoGoMcall = UnwindInfo{Flags: support.UnwindFlagCommand,
+	Param: support.UnwindCommandGoMcall}
+
 // UnwindInfoLR contains the description to unwind ARM64 function without a frame (LR only)
 var UnwindInfoLR = UnwindInfo{
 	BaseReg:    support.UnwindRegSp,

processmanager/execinfomanager/manager.go

@@ -126,9 +126,11 @@ func NewExecutableInfoManager(
 	}
 
 	interpreterLoaders = append(interpreterLoaders, apmint.Loader)
-	if includeTracers.Has(types.Labels) {
-		interpreterLoaders = append(interpreterLoaders, golabels.Loader)
-	}
+
+	// Register golabels loader. The native unwinder needs Go runtime
+	// offsets (m, curg, g.sched) to cross the systemstack boundary,
+	// to perform Go stack unwinding.
+	interpreterLoaders = append(interpreterLoaders, golabels.Loader)
 
 	deferredFileIDs, err := lru.NewSynced[host.FileID, libpf.Void](deferredFileIDSize,
 		func(id host.FileID) uint32 { return uint32(id) })

support/ebpf/errors.h

@@ -160,8 +160,8 @@ typedef enum ErrorCode {
   // Native: Unable to read the IRQ stack link
   ERR_NATIVE_CHASE_IRQ_STACK_LINK = 4010,
 
-  // Native: Unexpectedly encountered a kernel mode pointer while attempting to unwind user-mode
-  // stack
+  // Native: Unexpectedly encountered a kernel mode pointer while attempting
+  // to unwind user-mode stack
   ERR_NATIVE_UNEXPECTED_KERNEL_ADDRESS = 4011,
 
   // Native: Unable to locate the PID page mapping for the current instruction pointer
@@ -222,7 +222,19 @@ typedef enum ErrorCode {
   ERR_BEAM_MODULES_READ_FAILURE = 7005,
 
   // BEAM: Ran out of iterations searching for the current code header
-  ERR_BEAM_RANGE_SEARCH_EXHAUSTED = 7006
+  ERR_BEAM_RANGE_SEARCH_EXHAUSTED = 7006,
+
+  // Go: No Go process info available for mcall unwinding
+  ERR_GO_MCALL_NO_GO_OFFSETS = 8000,
+
+  // Go: Failed to resolve the user goroutine during mcall unwinding
+  ERR_GO_MCALL_RESOLVE_GOROUTINE = 8001,
+
+  // Go: Failed to read gobuf fields from the goroutine during mcall unwinding
+  ERR_GO_MCALL_READ_GOBUF = 8002,
+
+  // Go: Gobuf sp/pc is zero during mcall unwinding (sched not yet populated)
+  ERR_GO_MCALL_GOBUF_NOT_POPULATED = 8003
 } ErrorCode;
 
 #endif // OPTI_ERRORS_H

support/ebpf/go_runtime.h

@@ -0,0 +1,93 @@
+// This file contains helpers for reading Go runtime structures from eBPF programs.
+
+#ifndef OPTI_GO_RUNTIME_H
+#define OPTI_GO_RUNTIME_H
+
+#include "bpfdefs.h"
+#include "tsd.h"
+#include "types.h"
+
+// go_get_g_ptr reads the current G (goroutine) pointer from thread-local storage.
+// TLS always contains the G that is currently executing on the thread. During
+// systemstack/mcall, this is g0 (the system goroutine) since we are on the
+// system stack.
+//
+// On aarch64, the resolution path depends on whether the binary actually uses
+// cgo at runtime, which is not the same as buildinfo CGO_ENABLED:
+//
+//   - Pure-Go binaries (no `import "C"`): runtime.iscgo is false. The Go runtime
+//     never initialises TPIDR_EL0 for its threads ([1]), and load_g keeps g in R28.
+//   - Cgo binaries: libc initialises TPIDR_EL0 via pthread_create; load_g reads
+//     g from *(TPIDR_EL0 + tls_g_offset).
+//
+// The userspace TLS-offset extractor gates on buildinfo CGO_ENABLED only [2],
+// so it returns a non-zero offset for any binary built with CGO_ENABLED=1,
+// including pure-Go binaries where runtime.iscgo is false. 
+// To handle this safely we try TLS first and fall back to r28 if the read fails
+// or returns 0. R28 is the ABI-reserved register for the current goroutine
+// on aarch64 [3] and is always populated while executing pure Go code.
+//
+// [1] https://github.com/golang/go/blame/0259df17feb288f1e24517516939b67876c2627b/src/runtime/sys_linux_arm64.s#L705
+// [2] https://github.com/open-telemetry/opentelemetry-ebpf-profiler/blob/abd95fe39bdfcd00c8079d152123d38f459a6ff0/libpf/pfelf/file.go#L615
+// [3] https://github.com/golang/go/blob/0259df17feb288f1e24517516939b67876c2627b/src/cmd/compile/abi-internal.md?plain=1#L549
+static inline EBPF_INLINE u64 go_get_g_ptr(struct GoLabelsOffsets *offs, UnwindState *state)
+{
+#if defined(__x86_64__)
+  (void)state;
+#endif
+  u64 g_addr     = 0;
+  void *tls_base = NULL;
+  if (tsd_get_base(&tls_base) < 0) {
+    DEBUG_PRINT("cl: failed to get tsd base; can't read g_addr");
+    return 0;
+  }
+  DEBUG_PRINT(
+    "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset);
+
+  if (offs->tls_offset != 0 && tls_base != NULL) {
+    if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) {
+      DEBUG_PRINT("cl: failed to read g_addr via TLS, tls_base(%lx)", (unsigned long)tls_base);
+      g_addr = 0;
+    }
+  }
+
+#if defined(__aarch64__)
+  // Fallback to r28 when TLS is either not configured (pure-Go binary or
+  // mis-detected as cgo at build time) or when the TLS read failed. On
+  // aarch64 r28 holds the current g while executing Go runtime code.
+  if (g_addr == 0) {
+    g_addr = state->r28;
+    DEBUG_PRINT("cl: g_addr fallback via r28 = 0x%lx", (unsigned long)g_addr);
+  }
+#elif defined(__x86_64__)
+  if (g_addr == 0) {
+    DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64");
+    return 0;
+  }
+#endif
+
+  DEBUG_PRINT("cl: g_addr 0x%lx", (unsigned long)g_addr);
+  return g_addr;
+}
+
+// go_get_m_ptr reads the M (machine/OS thread) pointer for the current goroutine.
+// It does so by reading the G (goroutine) pointer from thread-local storage,
+// then following the g.m pointer.
+static inline EBPF_INLINE void *go_get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *state)
+{
+  u64 g_addr = go_get_g_ptr(offs, state);
+  if (!g_addr) {
+    return NULL;
+  }
+
+  DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset);
+  void *m_ptr_addr;
+  if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) {
+    DEBUG_PRINT("cl: failed m_ptr_addr");
+    return NULL;
+  }
+  DEBUG_PRINT("cl: m_ptr_addr 0x%lx", (unsigned long)m_ptr_addr);
+  return m_ptr_addr;
+}
+
+#endif // OPTI_GO_RUNTIME_H

support/ebpf/interpreter_dispatcher.ebpf.c

@@ -3,6 +3,7 @@
 // perf event and will call the appropriate tracer for a given process
 
 #include "bpfdefs.h"
+#include "go_runtime.h"
 #include "kernel.h"
 #include "tracemgmt.h"
 #include "tsd.h"
@@ -137,44 +138,6 @@ struct apm_int_procs_t {
 // filter_error_frames is set during load time.
 BPF_RODATA_VAR(bool, filter_error_frames, false)
 
-static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UNUSED UnwindState *state)
-{
-  u64 g_addr     = 0;
-  void *tls_base = NULL;
-  if (tsd_get_base(&tls_base) < 0) {
-    DEBUG_PRINT("cl: failed to get tsd base; can't read m_ptr");
-    return NULL;
-  }
-  DEBUG_PRINT(
-    "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset);
-
-  if (offs->tls_offset == 0) {
-#if defined(__aarch64__)
-    // On aarch64 for !iscgo programs the g is only stored in r28 register.
-    g_addr = state->r28;
-#elif defined(__x86_64__)
-    DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64");
-    return NULL;
-#endif
-  }
-
-  if (g_addr == 0) {
-    if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) {
-      DEBUG_PRINT("cl: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base);
-      return NULL;
-    }
-  }
-
-  DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset);
-  void *m_ptr_addr;
-  if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) {
-    DEBUG_PRINT("cl: failed m_ptr_addr");
-    return NULL;
-  }
-  DEBUG_PRINT("cl: m_ptr_addr 0x%lx", (unsigned long)m_ptr_addr);
-  return m_ptr_addr;
-}
-
 static EBPF_INLINE void maybe_add_go_custom_labels(struct pt_regs *ctx, PerCPURecord *record)
 {
   u32 pid                  = record->trace.pid;
@@ -184,7 +147,7 @@ static EBPF_INLINE void maybe_add_go_custom_labels(struct pt_regs *ctx, PerCPURe
     return;
   }
 
-  void *m_ptr_addr = get_m_ptr(offsets, &record->state);
+  void *m_ptr_addr = go_get_m_ptr(offsets, &record->state);
   if (!m_ptr_addr) {
     return;
   }