diff --git a/interpreter/golabels/runtime_data.go b/interpreter/golabels/runtime_data.go index 985dd4b21..7b38a15b6 100644 --- a/interpreter/golabels/runtime_data.go +++ b/interpreter/golabels/runtime_data.go @@ -27,16 +27,29 @@ func getOffsets(vers string) support.GoLabelsOffsets { Hmap_log2_bucket_count: 0, // https://github.com/golang/go/blob/6885bad7dd86880be6929c0/src/runtime/map.go#L118 Hmap_buckets: 0, + // g.sched is a gobuf struct immediately following g.m (offset 48 + 8 = 56). + // gobuf.sp is the first field (offset 0 in gobuf). + // https://github.com/golang/go/blob/80e2e474b8d9124d03b744f4e2da099a4eec5957/src/runtime/runtime2.go#L311 + Sched_sp: 56, + // Offsets of pc and bp within gobuf, relative to gobuf.sp. + // gobuf.pc is always at offset 8 (second field in gobuf). + Sched_pc_off: 8, + // gobuf.bp is at offset 48 within gobuf in go1.24 and earlier. In go1.25 and later, + // it is at offset 40 because of ret field removal. + // go1.25: https://github.com/golang/go/blob/6e676ab2b809d46623acb5988248d95d1eb7939c/src/runtime/runtime2.go#L315 + Sched_bp_off: 48, } // Version enforcement takes place in the Loader function. if version.Compare(vers, "go1.26") >= 0 { offsets.Curg = 184 offsets.Labels = 352 + offsets.Sched_bp_off = 40 return offsets } else if version.Compare(vers, "go1.25") >= 0 { offsets.Curg = 184 offsets.Labels = 344 + offsets.Sched_bp_off = 40 return offsets } else if version.Compare(vers, "go1.24") >= 0 { offsets.Labels = 352 diff --git a/metrics/ids.go b/metrics/ids.go index 76ed4dabe..f2faef4c6 100644 --- a/metrics/ids.go +++ b/metrics/ids.go @@ -650,6 +650,24 @@ const ( // Number of failures to read TLS variables via the DTV IDUnwindErrBadDTVRead = 286 + // Number of attempted Go mcall stack-switch unwinds + IDUnwindGoMcallAttempts = 287 + + // Number of successful Go mcall stack-switch unwinds + IDUnwindGoMcallSuccess = 288 + + // Number of Go mcall unwind failures due to missing Go offsets + IDUnwindGoMcallErrNoGoOffsets = 289 + + // Number of Go mcall unwind failures due to goroutine resolution + IDUnwindGoMcallErrResolveGoroutine = 290 + + // Number of Go mcall unwind failures due to gobuf read errors + IDUnwindGoMcallErrReadGobuf = 291 + + // Number of Go mcall unwind failures due to unpopulated gobuf + IDUnwindGoMcallErrGobufNotPopulated = 292 + // max number of ID values, keep this as *last entry* - IDMax = 287 + IDMax = 293 ) diff --git a/metrics/metrics.json b/metrics/metrics.json index 1021d7414..ef9aea76c 100644 --- a/metrics/metrics.json +++ b/metrics/metrics.json @@ -2096,5 +2096,47 @@ "name": "UnwindErrBadDTVRead", "field": "bpf.errors.bad_dtv_read", "id": 286 + }, + { + "description": "Number of attempted Go mcall stack-switch unwinds", + "type": "counter", + "name": "UnwindGoMcallAttempts", + "field": "bpf.go.mcall_attempts", + "id": 287 + }, + { + "description": "Number of successful Go mcall stack-switch unwinds", + "type": "counter", + "name": "UnwindGoMcallSuccess", + "field": "bpf.go.mcall_success", + "id": 288 + }, + { + "description": "Number of Go mcall unwind failures due to missing Go offsets", + "type": "counter", + "name": "UnwindGoMcallErrNoGoOffsets", + "field": "bpf.go.errors.mcall_no_go_offsets", + "id": 289 + }, + { + "description": "Number of Go mcall unwind failures due to goroutine resolution", + "type": "counter", + "name": "UnwindGoMcallErrResolveGoroutine", + "field": "bpf.go.errors.mcall_resolve_goroutine", + "id": 290 + }, + { + "description": "Number of Go mcall unwind failures due to gobuf read errors", + "type": "counter", + "name": "UnwindGoMcallErrReadGobuf", + "field": "bpf.go.errors.mcall_read_gobuf", + "id": 291 + }, + { + "description": "Number of Go mcall unwind failures due to unpopulated gobuf", + "type": "counter", + "name": "UnwindGoMcallErrGobufNotPopulated", + "field": "bpf.go.errors.mcall_gobuf_not_populated", + "id": 292 } ] diff --git a/nativeunwind/elfunwindinfo/elfgopclntab.go b/nativeunwind/elfunwindinfo/elfgopclntab.go index 8fec6ad75..e9ca4b601 100644 --- a/nativeunwind/elfunwindinfo/elfgopclntab.go +++ b/nativeunwind/elfunwindinfo/elfgopclntab.go @@ -31,10 +31,12 @@ var goFunctionsStopDelta = map[string]*sdtypes.UnwindInfo{ "runtime.mstart": &sdtypes.UnwindInfoStop, // topmost for the go runtime main stacks "runtime.goexit": &sdtypes.UnwindInfoStop, // return address in all goroutine stacks - // stack switch functions that would need special handling for further unwinding. - // See PF-1101. - "runtime.mcall": &sdtypes.UnwindInfoStop, - "runtime.systemstack": &sdtypes.UnwindInfoStop, + // Stack switch functions: systemstack preserves the frame pointer chain across + // the g0/user stack boundary, so standard FP unwinding traverses it naturally. + // mcall clears BP/R29 before calling fn, breaking the FP chain - it needs a + // custom command that reads gobuf.{pc, sp, bp} directly. + "runtime.systemstack": &sdtypes.UnwindInfoFramePointer, + "runtime.mcall": &sdtypes.UnwindInfoGoMcall, // signal return frame "runtime.sigreturn": &sdtypes.UnwindInfoSignal, diff --git a/nativeunwind/stackdeltatypes/stackdeltatypes.go b/nativeunwind/stackdeltatypes/stackdeltatypes.go index 334e9fa75..c0bc9cf6f 100644 --- a/nativeunwind/stackdeltatypes/stackdeltatypes.go +++ b/nativeunwind/stackdeltatypes/stackdeltatypes.go @@ -46,6 +46,14 @@ var UnwindInfoFramePointer = UnwindInfo{Flags: support.UnwindFlagCommand, Param: support.UnwindCommandFramePointer, } +// UnwindInfoGoMcall is the stack delta info for runtime.mcall. +// When encountered during unwinding, the unwinder crosses from the g0 system +// stack to the goroutine stack by reading the caller's saved registers directly +// from gobuf.{pc, sp, bp}. If m.curg is nil (after dropg), the goroutine pointer +// is recovered from the g0 stack at *(g0.sched.sp - 8). +var UnwindInfoGoMcall = UnwindInfo{Flags: support.UnwindFlagCommand, + Param: support.UnwindCommandGoMcall} + // UnwindInfoLR contains the description to unwind ARM64 function without a frame (LR only) var UnwindInfoLR = UnwindInfo{ BaseReg: support.UnwindRegSp, diff --git a/processmanager/execinfomanager/manager.go b/processmanager/execinfomanager/manager.go index cf6c5364d..08d50526d 100644 --- a/processmanager/execinfomanager/manager.go +++ b/processmanager/execinfomanager/manager.go @@ -126,9 +126,11 @@ func NewExecutableInfoManager( } interpreterLoaders = append(interpreterLoaders, apmint.Loader) - if includeTracers.Has(types.Labels) { - interpreterLoaders = append(interpreterLoaders, golabels.Loader) - } + + // Register golabels loader. The native unwinder needs Go runtime + // offsets (m, curg, g.sched) to cross the systemstack boundary, + // to perform Go stack unwinding. + interpreterLoaders = append(interpreterLoaders, golabels.Loader) deferredFileIDs, err := lru.NewSynced[host.FileID, libpf.Void](deferredFileIDSize, func(id host.FileID) uint32 { return uint32(id) }) diff --git a/support/ebpf/errors.h b/support/ebpf/errors.h index 0da717277..ffd8c6008 100644 --- a/support/ebpf/errors.h +++ b/support/ebpf/errors.h @@ -160,8 +160,8 @@ typedef enum ErrorCode { // Native: Unable to read the IRQ stack link ERR_NATIVE_CHASE_IRQ_STACK_LINK = 4010, - // Native: Unexpectedly encountered a kernel mode pointer while attempting to unwind user-mode - // stack + // Native: Unexpectedly encountered a kernel mode pointer while attempting + // to unwind user-mode stack ERR_NATIVE_UNEXPECTED_KERNEL_ADDRESS = 4011, // Native: Unable to locate the PID page mapping for the current instruction pointer @@ -222,7 +222,19 @@ typedef enum ErrorCode { ERR_BEAM_MODULES_READ_FAILURE = 7005, // BEAM: Ran out of iterations searching for the current code header - ERR_BEAM_RANGE_SEARCH_EXHAUSTED = 7006 + ERR_BEAM_RANGE_SEARCH_EXHAUSTED = 7006, + + // Go: No Go process info available for mcall unwinding + ERR_GO_MCALL_NO_GO_OFFSETS = 8000, + + // Go: Failed to resolve the user goroutine during mcall unwinding + ERR_GO_MCALL_RESOLVE_GOROUTINE = 8001, + + // Go: Failed to read gobuf fields from the goroutine during mcall unwinding + ERR_GO_MCALL_READ_GOBUF = 8002, + + // Go: Gobuf sp/pc is zero during mcall unwinding (sched not yet populated) + ERR_GO_MCALL_GOBUF_NOT_POPULATED = 8003 } ErrorCode; #endif // OPTI_ERRORS_H diff --git a/support/ebpf/go_runtime.h b/support/ebpf/go_runtime.h new file mode 100644 index 000000000..22d9456e3 --- /dev/null +++ b/support/ebpf/go_runtime.h @@ -0,0 +1,93 @@ +// This file contains helpers for reading Go runtime structures from eBPF programs. + +#ifndef OPTI_GO_RUNTIME_H +#define OPTI_GO_RUNTIME_H + +#include "bpfdefs.h" +#include "tsd.h" +#include "types.h" + +// go_get_g_ptr reads the current G (goroutine) pointer from thread-local storage. +// TLS always contains the G that is currently executing on the thread. During +// systemstack/mcall, this is g0 (the system goroutine) since we are on the +// system stack. +// +// On aarch64, the resolution path depends on whether the binary actually uses +// cgo at runtime, which is not the same as buildinfo CGO_ENABLED: +// +// - Pure-Go binaries (no `import "C"`): runtime.iscgo is false. The Go runtime +// never initialises TPIDR_EL0 for its threads ([1]), and load_g keeps g in R28. +// - Cgo binaries: libc initialises TPIDR_EL0 via pthread_create; load_g reads +// g from *(TPIDR_EL0 + tls_g_offset). +// +// The userspace TLS-offset extractor gates on buildinfo CGO_ENABLED only [2], +// so it returns a non-zero offset for any binary built with CGO_ENABLED=1, +// including pure-Go binaries where runtime.iscgo is false. +// To handle this safely we try TLS first and fall back to r28 if the read fails +// or returns 0. R28 is the ABI-reserved register for the current goroutine +// on aarch64 [3] and is always populated while executing pure Go code. +// +// [1] https://github.com/golang/go/blame/0259df17feb288f1e24517516939b67876c2627b/src/runtime/sys_linux_arm64.s#L705 +// [2] https://github.com/open-telemetry/opentelemetry-ebpf-profiler/blob/abd95fe39bdfcd00c8079d152123d38f459a6ff0/libpf/pfelf/file.go#L615 +// [3] https://github.com/golang/go/blob/0259df17feb288f1e24517516939b67876c2627b/src/cmd/compile/abi-internal.md?plain=1#L549 +static inline EBPF_INLINE u64 go_get_g_ptr(struct GoLabelsOffsets *offs, UnwindState *state) +{ +#if defined(__x86_64__) + (void)state; +#endif + u64 g_addr = 0; + void *tls_base = NULL; + if (tsd_get_base(&tls_base) < 0) { + DEBUG_PRINT("cl: failed to get tsd base; can't read g_addr"); + return 0; + } + DEBUG_PRINT( + "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset); + + if (offs->tls_offset != 0 && tls_base != NULL) { + if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) { + DEBUG_PRINT("cl: failed to read g_addr via TLS, tls_base(%lx)", (unsigned long)tls_base); + g_addr = 0; + } + } + +#if defined(__aarch64__) + // Fallback to r28 when TLS is either not configured (pure-Go binary or + // mis-detected as cgo at build time) or when the TLS read failed. On + // aarch64 r28 holds the current g while executing Go runtime code. + if (g_addr == 0) { + g_addr = state->r28; + DEBUG_PRINT("cl: g_addr fallback via r28 = 0x%lx", (unsigned long)g_addr); + } +#elif defined(__x86_64__) + if (g_addr == 0) { + DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64"); + return 0; + } +#endif + + DEBUG_PRINT("cl: g_addr 0x%lx", (unsigned long)g_addr); + return g_addr; +} + +// go_get_m_ptr reads the M (machine/OS thread) pointer for the current goroutine. +// It does so by reading the G (goroutine) pointer from thread-local storage, +// then following the g.m pointer. +static inline EBPF_INLINE void *go_get_m_ptr(struct GoLabelsOffsets *offs, UnwindState *state) +{ + u64 g_addr = go_get_g_ptr(offs, state); + if (!g_addr) { + return NULL; + } + + DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset); + void *m_ptr_addr; + if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) { + DEBUG_PRINT("cl: failed m_ptr_addr"); + return NULL; + } + DEBUG_PRINT("cl: m_ptr_addr 0x%lx", (unsigned long)m_ptr_addr); + return m_ptr_addr; +} + +#endif // OPTI_GO_RUNTIME_H diff --git a/support/ebpf/interpreter_dispatcher.ebpf.c b/support/ebpf/interpreter_dispatcher.ebpf.c index ae9a1c9a7..4ab11b2d9 100644 --- a/support/ebpf/interpreter_dispatcher.ebpf.c +++ b/support/ebpf/interpreter_dispatcher.ebpf.c @@ -3,6 +3,7 @@ // perf event and will call the appropriate tracer for a given process #include "bpfdefs.h" +#include "go_runtime.h" #include "kernel.h" #include "tracemgmt.h" #include "tsd.h" @@ -137,44 +138,6 @@ struct apm_int_procs_t { // filter_error_frames is set during load time. BPF_RODATA_VAR(bool, filter_error_frames, false) -static EBPF_INLINE void *get_m_ptr(struct GoLabelsOffsets *offs, UNUSED UnwindState *state) -{ - u64 g_addr = 0; - void *tls_base = NULL; - if (tsd_get_base(&tls_base) < 0) { - DEBUG_PRINT("cl: failed to get tsd base; can't read m_ptr"); - return NULL; - } - DEBUG_PRINT( - "cl: read tsd_base at 0x%lx, g offset: %d", (unsigned long)tls_base, offs->tls_offset); - - if (offs->tls_offset == 0) { -#if defined(__aarch64__) - // On aarch64 for !iscgo programs the g is only stored in r28 register. - g_addr = state->r28; -#elif defined(__x86_64__) - DEBUG_PRINT("cl: TLS offset for g pointer missing for amd64"); - return NULL; -#endif - } - - if (g_addr == 0) { - if (bpf_probe_read_user(&g_addr, sizeof(void *), (void *)((s64)tls_base + offs->tls_offset))) { - DEBUG_PRINT("cl: failed to read g_addr, tls_base(%lx)", (unsigned long)tls_base); - return NULL; - } - } - - DEBUG_PRINT("cl: reading m_ptr_addr at 0x%lx + 0x%x", (unsigned long)g_addr, offs->m_offset); - void *m_ptr_addr; - if (bpf_probe_read_user(&m_ptr_addr, sizeof(void *), (void *)(g_addr + offs->m_offset))) { - DEBUG_PRINT("cl: failed m_ptr_addr"); - return NULL; - } - DEBUG_PRINT("cl: m_ptr_addr 0x%lx", (unsigned long)m_ptr_addr); - return m_ptr_addr; -} - static EBPF_INLINE void maybe_add_go_custom_labels(struct pt_regs *ctx, PerCPURecord *record) { u32 pid = record->trace.pid; @@ -184,7 +147,7 @@ static EBPF_INLINE void maybe_add_go_custom_labels(struct pt_regs *ctx, PerCPURe return; } - void *m_ptr_addr = get_m_ptr(offsets, &record->state); + void *m_ptr_addr = go_get_m_ptr(offsets, &record->state); if (!m_ptr_addr) { return; } diff --git a/support/ebpf/native_stack_trace.ebpf.c b/support/ebpf/native_stack_trace.ebpf.c index dc3049f98..bfc735b36 100644 --- a/support/ebpf/native_stack_trace.ebpf.c +++ b/support/ebpf/native_stack_trace.ebpf.c @@ -1,6 +1,9 @@ #include "bpfdefs.h" +#include "extmaps.h" #include "frametypes.h" +#include "go_runtime.h" #include "tracemgmt.h" +#include "tsd.h" #include "types.h" // with_debug_output is set during load time. @@ -298,18 +301,225 @@ unwind_calc_register_with_deref(UnwindState *state, u8 baseReg, s32 param, bool } #endif +// go_resolve_mcall_goroutine resolves the user goroutine for mcall unwinding. +// First tries m.curg (valid before dropg). If nil, falls back +// to reading the goroutine pointer from g0's stack at *(g0.sched.sp - 8), where +// mcall deterministically places it (via PUSHQ on AMD64, or ABIInternal arg spill +// on ARM64). The candidate is validated to guard +// against stale data on ARM64 when fn didn't spill. +// +// Returns the user goroutine address, or 0 on failure (caller should STOP). +static EBPF_INLINE u64 +go_resolve_mcall_goroutine(struct GoLabelsOffsets *offs, PerCPURecord *record) +{ + UnwindState *state = &record->state; + u64 g0_addr = go_get_g_ptr(offs, state); + if (!g0_addr) { + return 0; + } + + if (offs->sched_sp != offs->m_offset + sizeof(u64)) { + DEBUG_PRINT("GO_MCALL: unexpected g.m/g.sched.sp layout"); + return 0; + } + + u64 *go_scratch = (u64 *)record->goUnwindScratch.gobuf; + if (bpf_probe_read_user(go_scratch, sizeof(u64) * 2, (void *)(g0_addr + offs->m_offset))) { + return 0; + } + u64 g0_m_ptr = go_scratch[0]; + u64 g0_sched_sp = go_scratch[1]; + if (!g0_m_ptr) { + return 0; + } + + // Try m.curg first (valid before dropg, cold path). + u64 curg = 0; + if (bpf_probe_read_user(&curg, sizeof(curg), (void *)(g0_m_ptr + offs->curg))) { + return 0; + } + if (curg) { + DEBUG_PRINT("GO_MCALL: found curg via m.curg = 0x%lx", (unsigned long)curg); + return curg; + } + + // m.curg is nil (dropg was called). Recover old_g from g0 stack. + if (!g0_sched_sp) { + DEBUG_PRINT("GO_MCALL: g0.sched.sp is 0"); + return 0; + } + + u64 candidate = 0; + if (bpf_probe_read_user(&candidate, sizeof(candidate), (void *)(g0_sched_sp - 8))) { + DEBUG_PRINT("GO_MCALL: failed to read *(g0.sched.sp - 8)"); + return 0; + } + if (!candidate) { + DEBUG_PRINT("GO_MCALL: candidate goroutine is NULL"); + return 0; + } + + // Validate: the goroutine must still be parked (g.m == nil). + // After dropg(), g.m is set to nil. If the goroutine has been rescheduled on + // another M, g.m points to that M (non-nil) and its gobuf may have been + // overwritten by systemstack or another mechanism reading it would produce + // a wrong stack trace (e.g., systemstack_switch frames from another thread). + if (bpf_probe_read_user(go_scratch, sizeof(u64) * 2, (void *)(candidate + offs->m_offset))) { + DEBUG_PRINT("GO_MCALL: failed to read candidate.m"); + return 0; + } + u64 cand_m_ptr = go_scratch[0]; + u64 cand_sched_sp = go_scratch[1]; + if (cand_m_ptr) { + DEBUG_PRINT( + "GO_MCALL: candidate.m is non-nil (0x%lx), goroutine rescheduled on another M", + (unsigned long)cand_m_ptr); + return 0; + } + + // Also check that gobuf.sp is non-zero (populated by mcall before the switch). + if (!cand_sched_sp) { + DEBUG_PRINT("GO_MCALL: candidate.sched.sp is 0"); + return 0; + } + + DEBUG_PRINT("GO_MCALL: recovered goroutine 0x%lx from g0 stack slot", (unsigned long)candidate); + return candidate; +} + +// go_unwind_mcall crosses the Go mcall stack-switch boundary by reading the +// caller's saved registers from gobuf.{pc, sp, bp}. Unlike systemstack (which +// preserves the frame pointer chain and uses FRAME_POINTER unwinding), mcall +// clears BP/R29 before calling fn, so the FP chain is broken and we must read +// the gobuf fields directly. +// +// mcall saves the caller's actual registers into gobuf before switching to g0: +// +// AMD64 (asm_amd64.s): +// MOVQ 8(BX), BX // gobuf.pc = caller's return address +// LEAQ fn+0(FP), BX // gobuf.sp = caller's SP +// MOVQ (BP), BX // gobuf.bp = caller's FP +// +// After saving gobuf, mcall switches to g0 and calls fn(old_g): +// +// MOVQ R14, AX // AX = old_g (user goroutine) +// MOVQ SI, R14 // R14 = g0 +// MOVQ R14, FS:0xfffffff8 // TLS = g0 +// MOVQ (g_sched+gobuf_sp)(R14), SP // SP = g0.sched.sp +// MOVQ $0, BP // clear frame pointer +// PUSHQ AX // *(g0.sched.sp - 8) = old_g +// CALL R12 // fn(old_g) +// https://github.com/golang/go/blob/3901409b5d0fb7c85a3e6730a59943cc93b2835c/src/runtime/asm_amd64.s#L459 +// +// ARM64 (asm_arm64.s, NOSPLIT|NOFRAME - no prologue): +// MOVD RSP, R0 +// MOVD R0, (g_sched+gobuf_sp)(g) // gobuf.sp = caller's SP +// MOVD R29, (g_sched+gobuf_bp)(g) // gobuf.bp = caller's FP +// MOVD R30, (g_sched+gobuf_pc)(g) // gobuf.pc = LR = caller's return addr +// +// After saving gobuf, mcall switches to g0 and calls fn(old_g): +// +// MOVD (g_sched+gobuf_sp)(g), R0 +// MOVD R0, RSP // RSP = g0.sched.sp +// MOVD ZR, R29 // clear frame pointer +// MOVD R3, R0 // R0 = old_g (fn's argument) +// MOVD ZR, -16(RSP) +// SUB $16, RSP // allocate 16-byte arg space +// BL (R4) // fn(old_g), entry SP = g0.sched.sp - 16 +// https://github.com/golang/go/blob/3901409b5d0fb7c85a3e6730a59943cc93b2835c/src/runtime/asm_arm64.s#L193 +// +// fn (park_m, goexit0, etc.) typically calls dropg() which sets m.curg to nil. +// go_resolve_mcall_goroutine() handles this by falling back to +// *(g0.sched.sp - 8) where: +// AMD64: PUSHQ AX deterministically placed old_g +// ARM64: fn's ABIInternal prologue spills R0 (old_g) to its arg area +// at fn_entry_SP + 8 = g0.sched.sp - 16 + 8 = g0.sched.sp - 8 +// (verified via DWARF: DW_OP_fbreg(+8) = CFA+8) +// +// Returns ERR_OK if the transition succeeded and state was updated. +// Returns a specific ErrorCode on failure (caller should STOP). +static EBPF_INLINE ErrorCode go_unwind_mcall(UnwindState *state) +{ + increment_metric(metricID_UnwindGoMcallAttempts); + + PerCPURecord *record = get_per_cpu_record(); + if (!record) { + return ERR_UNREACHABLE; + } + u32 pid = record->trace.pid; + GoLabelsOffsets *go_offs = bpf_map_lookup_elem(&go_labels_procs, &pid); + if (!go_offs || !go_offs->sched_sp) { + increment_metric(metricID_UnwindGoMcallErrNoGoOffsets); + return ERR_GO_MCALL_NO_GO_OFFSETS; + } + u64 curg = go_resolve_mcall_goroutine(go_offs, record); + if (!curg) { + DEBUG_PRINT("GO_MCALL: could not resolve user goroutine, stopping"); + increment_metric(metricID_UnwindGoMcallErrResolveGoroutine); + return ERR_GO_MCALL_RESOLVE_GOROUTINE; + } + + // Read gobuf.{sp, pc, bp} in a single bpf_probe_read_user into the PerCPURecord + // scratch buffer. See GoUnwindScratchSpace in types.h. Gobuf layout in runtime_data.go. + u8 *gobuf = record->goUnwindScratch.gobuf; + if (bpf_probe_read_user( + gobuf, sizeof(record->goUnwindScratch.gobuf), (void *)(curg + go_offs->sched_sp))) { + DEBUG_PRINT("GO_MCALL: failed to read gobuf, stopping"); + increment_metric(metricID_UnwindGoMcallErrReadGobuf); + return ERR_GO_MCALL_READ_GOBUF; + } + // sched_pc_off and sched_bp_off are u8 offsets relative to gobuf.sp, stored + // in GoLabelsOffsets. Using u8 fields (not u32 subtractions) lets the BPF + // verifier bound them naturally. + if ( + go_offs->sched_bp_off + sizeof(u64) > sizeof(record->goUnwindScratch.gobuf) || + go_offs->sched_pc_off + sizeof(u64) > sizeof(record->goUnwindScratch.gobuf)) { + increment_metric(metricID_UnwindGoMcallErrReadGobuf); + return ERR_GO_MCALL_READ_GOBUF; + } + u64 saved_sp = *(u64 *)(gobuf); + u64 saved_pc = *(u64 *)(gobuf + go_offs->sched_pc_off); + u64 saved_bp = *(u64 *)(gobuf + go_offs->sched_bp_off); + + if (!saved_sp || !saved_pc) { + DEBUG_PRINT( + "GO_MCALL: gobuf not populated (sp=0x%lx pc=0x%lx), stopping", + (unsigned long)saved_sp, + (unsigned long)saved_pc); + increment_metric(metricID_UnwindGoMcallErrGobufNotPopulated); + return ERR_GO_MCALL_GOBUF_NOT_POPULATED; + } + DEBUG_PRINT( + "GO_MCALL: recovered context: pc=0x%lx, sp=0x%lx, fp=0x%lx", + (unsigned long)saved_pc, + (unsigned long)saved_sp, + (unsigned long)saved_bp); + state->sp = saved_sp; + state->fp = saved_bp; +#if defined(__aarch64__) + state->pc = normalize_pac_ptr(saved_pc); + state->lr = 0; + state->r28 = curg; +#else + state->pc = saved_pc; +#endif + unwinder_mark_nonleaf_frame(state); + increment_metric(metricID_UnwindGoMcallSuccess); + return ERR_OK; +} + // Stack unwinding in the absence of frame pointers can be a bit involved, so // this comment explains what the following code does. // // One begins unwinding a frame somewhere in the middle of execution. // On x86_64, registers RIP (PC), RSP (SP), and RBP (FP) are available. // -// This function resolves a "stack delta" command from from our internal maps. +// This function resolves a "stack delta" command from our internal maps. // This stack delta refers to a rule on how to unwind the state. In the simple // case it just provides SP delta and potentially offset from where to recover // FP value. See unwind_calc_register[_with_deref]() on the expressions supported. // -// The function sets the bool pointed to by the given `stop` pointer to `false` +// The function sets the bool pointed to by the given `stop` pointer to `true` // if the main ebpf unwinder should exit. This is the case if the current PC // is marked with UNWIND_COMMAND_STOP which marks entry points (main function, // thread spawn function, signal handlers, ...). @@ -372,6 +582,14 @@ static EBPF_INLINE ErrorCode unwind_one_frame(PerCPURecord *record, bool *stop) goto err_native_pc_read; } goto frame_ok; + case UNWIND_COMMAND_GO_MCALL: { + ErrorCode mcall_err = go_unwind_mcall(state); + if (mcall_err == ERR_OK) { + goto frame_ok; + } + *stop = true; + return ERR_OK; + } default: return ERR_UNREACHABLE; } } else { @@ -475,6 +693,14 @@ static EBPF_INLINE ErrorCode unwind_one_frame(PerCPURecord *record, bool *stop) goto err_native_pc_read; } goto frame_ok; + case UNWIND_COMMAND_GO_MCALL: { + ErrorCode mcall_err = go_unwind_mcall(state); + if (mcall_err == ERR_OK) { + goto frame_ok; + } + *stop = true; + return ERR_OK; + } default: return ERR_UNREACHABLE; } } diff --git a/support/ebpf/tracer.ebpf.amd64 b/support/ebpf/tracer.ebpf.amd64 index c745bd234..abae69b1f 100644 Binary files a/support/ebpf/tracer.ebpf.amd64 and b/support/ebpf/tracer.ebpf.amd64 differ diff --git a/support/ebpf/tracer.ebpf.arm64 b/support/ebpf/tracer.ebpf.arm64 index a964f079d..b3025408e 100644 Binary files a/support/ebpf/tracer.ebpf.arm64 and b/support/ebpf/tracer.ebpf.arm64 differ diff --git a/support/ebpf/tsd.h b/support/ebpf/tsd.h index 28accaf76..3f352120f 100644 --- a/support/ebpf/tsd.h +++ b/support/ebpf/tsd.h @@ -2,6 +2,7 @@ #define OPTI_TSD_H #include "bpfdefs.h" +#include "tracemgmt.h" // tpbase_offset is declared in native_stack_trace.ebpf.c extern u64 tpbase_offset; diff --git a/support/ebpf/types.h b/support/ebpf/types.h index 187caa82d..df2d961d0 100644 --- a/support/ebpf/types.h +++ b/support/ebpf/types.h @@ -328,6 +328,24 @@ enum { // number of failures to read TLS variables via the DTV metricID_UnwindErrBadDTVRead, + // number of attempted Go mcall stack-switch unwinds + metricID_UnwindGoMcallAttempts, + + // number of successful Go mcall stack-switch unwinds + metricID_UnwindGoMcallSuccess, + + // number of Go mcall unwind failures due to missing Go offsets + metricID_UnwindGoMcallErrNoGoOffsets, + + // number of Go mcall unwind failures due to goroutine resolution + metricID_UnwindGoMcallErrResolveGoroutine, + + // number of Go mcall unwind failures due to gobuf read errors + metricID_UnwindGoMcallErrReadGobuf, + + // number of Go mcall unwind failures due to unpopulated gobuf + metricID_UnwindGoMcallErrGobufNotPopulated, + // // Metric IDs above are for counters (cumulative values) // @@ -770,6 +788,14 @@ typedef struct V8UnwindScratchSpace { u8 code[96]; } V8UnwindScratchSpace; +// Container for additional scratch space needed by the Go unwinder. +typedef struct GoUnwindScratchSpace { + // The gobuf struct has a maximum size of 56 bytes (Go <= 1.24, with ret field). + // This buffer is used to read gobuf in a single bpf_probe_read_user call and + // extract fields at dynamic offsets. + u8 gobuf[56]; +} GoUnwindScratchSpace; + // Container for additional scratch space needed by the Python unwinder. typedef struct PythonUnwindScratchSpace { // Read buffer for storing the PyInterpreterFrame (PyFrameObject). @@ -805,6 +831,19 @@ typedef struct CustomLabelsState { void *go_m_ptr; } CustomLabelsState; +typedef struct GoLabelsOffsets { + u32 m_offset; + u32 curg; + u32 labels; + u32 hmap_count; + u32 hmap_log2_bucket_count; + u32 hmap_buckets; + s32 tls_offset; + u32 sched_sp; + u8 sched_pc_off; + u8 sched_bp_off; +} GoLabelsOffsets; + // Per-CPU info for the stack being built. This contains the stack as well as // meta-data on the number of eBPF tail-calls used so far to construct it. typedef struct PerCPURecord { @@ -842,7 +881,10 @@ typedef struct PerCPURecord { #elif defined(__aarch64__) u64 rt_regs[34]; #endif + // Scratch for Go unwinder + GoUnwindScratchSpace goUnwindScratch; }; + // Mask to indicate which unwinders are complete u32 unwindersDone; @@ -930,6 +972,8 @@ typedef struct StackDelta { #define UNWIND_COMMAND_SIGNAL 3 // Unwind using standard frame pointer #define UNWIND_COMMAND_FRAME_POINTER 4 +// Cross Go mcall boundary using goroutine saved context from gobuf fields directly. +#define UNWIND_COMMAND_GO_MCALL 5 // StackDeltaPageKey is the look up key for stack delta page map. typedef struct StackDeltaPageKey { @@ -1040,14 +1084,4 @@ typedef struct ApmIntProcInfo { u64 tls_offset; } ApmIntProcInfo; -typedef struct GoLabelsOffsets { - u32 m_offset; - u32 curg; - u32 labels; - u32 hmap_count; - u32 hmap_log2_bucket_count; - u32 hmap_buckets; - s32 tls_offset; -} GoLabelsOffsets; - #endif // OPTI_TYPES_H diff --git a/support/types.go b/support/types.go index 34a94bde5..5e5374f69 100644 --- a/support/types.go +++ b/support/types.go @@ -61,7 +61,7 @@ const ( const UnwindInfoMaxEntries = 0x4000 const ( - MetricIDBeginCumulative = 0x69 + MetricIDBeginCumulative = 0x6f ) const ( @@ -200,6 +200,10 @@ type GoLabelsOffsets struct { Hmap_log2_bucket_count uint32 Hmap_buckets uint32 Tls_offset int32 + Sched_sp uint32 + Sched_pc_off uint8 + Sched_bp_off uint8 + Pad_cgo_0 [2]byte } type HotspotProcInfo struct { Codecache_start uint64 @@ -363,6 +367,7 @@ const ( UnwindCommandPLT int32 = 0x2 UnwindCommandSignal int32 = 0x3 UnwindCommandFramePointer int32 = 0x4 + UnwindCommandGoMcall int32 = 0x5 UnwindDerefMask int32 = 0x7 UnwindDerefMultiplier int32 = 0x8 @@ -497,4 +502,10 @@ var MetricsTranslation = []metrics.MetricID{ 0x66: metrics.IDUnwindRubyErrReadRbasicFlags, 0x67: metrics.IDUnwindRubyErrCmeMaxEp, 0x68: metrics.IDUnwindErrBadDTVRead, + 0x69: metrics.IDUnwindGoMcallAttempts, + 0x6a: metrics.IDUnwindGoMcallSuccess, + 0x6b: metrics.IDUnwindGoMcallErrNoGoOffsets, + 0x6c: metrics.IDUnwindGoMcallErrResolveGoroutine, + 0x6d: metrics.IDUnwindGoMcallErrReadGobuf, + 0x6e: metrics.IDUnwindGoMcallErrGobufNotPopulated, } diff --git a/support/types_def.go b/support/types_def.go index 760b263f2..7d6a00a09 100644 --- a/support/types_def.go +++ b/support/types_def.go @@ -170,6 +170,7 @@ const ( UnwindCommandPLT int32 = C.UNWIND_COMMAND_PLT UnwindCommandSignal int32 = C.UNWIND_COMMAND_SIGNAL UnwindCommandFramePointer int32 = C.UNWIND_COMMAND_FRAME_POINTER + UnwindCommandGoMcall int32 = C.UNWIND_COMMAND_GO_MCALL // UnwindDeref handling from the C header file UnwindDerefMask int32 = C.UNWIND_DEREF_MASK @@ -307,4 +308,10 @@ var MetricsTranslation = []metrics.MetricID{ C.metricID_UnwindRubyErrReadRbasicFlags: metrics.IDUnwindRubyErrReadRbasicFlags, C.metricID_UnwindRubyErrCmeMaxEp: metrics.IDUnwindRubyErrCmeMaxEp, C.metricID_UnwindErrBadDTVRead: metrics.IDUnwindErrBadDTVRead, + C.metricID_UnwindGoMcallAttempts: metrics.IDUnwindGoMcallAttempts, + C.metricID_UnwindGoMcallSuccess: metrics.IDUnwindGoMcallSuccess, + C.metricID_UnwindGoMcallErrNoGoOffsets: metrics.IDUnwindGoMcallErrNoGoOffsets, + C.metricID_UnwindGoMcallErrResolveGoroutine: metrics.IDUnwindGoMcallErrResolveGoroutine, + C.metricID_UnwindGoMcallErrReadGobuf: metrics.IDUnwindGoMcallErrReadGobuf, + C.metricID_UnwindGoMcallErrGobufNotPopulated: metrics.IDUnwindGoMcallErrGobufNotPopulated, } diff --git a/tools/coredump/ebpfhelpers.go b/tools/coredump/ebpfhelpers.go index 117fbef0f..4fd5ffae8 100644 --- a/tools/coredump/ebpfhelpers.go +++ b/tools/coredump/ebpfhelpers.go @@ -110,7 +110,7 @@ func __bpf_map_lookup_elem(id C.u64, mapdef unsafe.Pointer, keyptr unsafe.Pointe case unsafe.Pointer(&C.dotnet_procs), unsafe.Pointer(&C.perl_procs), unsafe.Pointer(&C.php_procs), unsafe.Pointer(&C.py_procs), unsafe.Pointer(&C.hotspot_procs), unsafe.Pointer(&C.ruby_procs), - unsafe.Pointer(&C.v8_procs): + unsafe.Pointer(&C.v8_procs), unsafe.Pointer(&C.go_labels_procs): if innerMap, ok := ctx.maps[mapdef]; ok { if val, ok := innerMap[*(*C.u32)(keyptr)]; ok { return val @@ -138,7 +138,7 @@ func __bpf_map_lookup_elem(id C.u64, mapdef unsafe.Pointer, keyptr unsafe.Pointe } case unsafe.Pointer(&C.metrics), unsafe.Pointer(&C.report_events), unsafe.Pointer(&C.reported_pids), unsafe.Pointer(&C.pid_events), unsafe.Pointer(&C.inhibit_events), - unsafe.Pointer(&C.apm_int_procs), unsafe.Pointer(&C.go_labels_procs): + unsafe.Pointer(&C.apm_int_procs): return unsafe.Pointer(uintptr(0)) default: log.Errorf("Map at 0x%x not found", mapdef) diff --git a/tools/coredump/ebpfmaps.go b/tools/coredump/ebpfmaps.go index db9b83709..753d5d1a5 100644 --- a/tools/coredump/ebpfmaps.go +++ b/tools/coredump/ebpfmaps.go @@ -72,6 +72,9 @@ func (emc *ebpfMapsCoredump) UpdateProcData(t libpf.InterpreterType, pid libpf.P emc.ctx.addMap(unsafe.Pointer(&C.v8_procs), C.u32(pid), sliceBuffer(ptr, C.sizeof_V8ProcInfo)) case libpf.BEAM: emc.ctx.addMap(unsafe.Pointer(&C.beam_procs), C.u32(pid), sliceBuffer(ptr, C.sizeof_BEAMProcInfo)) + case libpf.GoLabels: + emc.ctx.addMap(unsafe.Pointer(&C.go_labels_procs), C.u32(pid), + sliceBuffer(ptr, C.sizeof_GoLabelsOffsets)) } return nil } @@ -94,6 +97,8 @@ func (emc *ebpfMapsCoredump) DeleteProcData(t libpf.InterpreterType, pid libpf.P emc.ctx.delMap(unsafe.Pointer(&C.v8_procs), C.u32(pid)) case libpf.BEAM: emc.ctx.delMap(unsafe.Pointer(&C.beam_procs), C.u32(pid)) + case libpf.GoLabels: + emc.ctx.delMap(unsafe.Pointer(&C.go_labels_procs), C.u32(pid)) } return nil } diff --git a/tools/coredump/testdata/amd64/go-mcallpark.json b/tools/coredump/testdata/amd64/go-mcallpark.json new file mode 100644 index 000000000..780609ac3 --- /dev/null +++ b/tools/coredump/testdata/amd64/go-mcallpark.json @@ -0,0 +1,118 @@ +{ + "coredump-ref": "7e454796ef114350d3bb35ec80fb794eda816ee4795e6bed9533582b9a626940", + "threads": [ + { + "lwp": 9537, + "frames": [ + "internal/runtime/syscall.Syscall6+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/internal/runtime/syscall/asm_linux_amd64.s:36", + "internal/runtime/syscall.EpollWait+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/internal/runtime/syscall/syscall_linux.go:34", + "runtime.netpoll+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/netpoll_epoll.go:120", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3728", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.park_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4267", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "runtime.gopark+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:510", + "main.parker+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 9543, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.park_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4267", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "runtime.gopark+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:510", + "main.parker+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 9542, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.park_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4267", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "runtime.gopark+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:510", + "main.parker+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 9541, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.park_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4267", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "runtime.gopark+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:510", + "main.parker+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 9540, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.park_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4267", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "runtime.gopark+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/chan.go:510", + "main.parker+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 9539, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:82", + "runtime.notetsleep_internal+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:90", + "runtime.notetsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:112", + "runtime.sysmon+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:6270", + "runtime.mstart1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1928", + "runtime.mstart0+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1890", + "runtime.mstart+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:396" + ] + } + ], + "modules": [ + { + "ref": "12fecf3742c47c5da69e900e7282601b6c41510f63175a6e3c4134d9d13d4601", + "local-path": "/home/ubuntu/ebpf/test-mcallpark" + } + ] +} diff --git a/tools/coredump/testdata/amd64/go-mcallstop.json b/tools/coredump/testdata/amd64/go-mcallstop.json new file mode 100644 index 000000000..95cd930bb --- /dev/null +++ b/tools/coredump/testdata/amd64/go-mcallstop.json @@ -0,0 +1,173 @@ +{ + "coredump-ref": "157325498d0330691b067551246338458beb737e18ff54379095b84c951b3694", + "threads": [ + { + "lwp": 14778, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "main.yielder+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallstop/main.go:14", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 14795, + "frames": [ + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14794, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14793, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14792, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14791, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14790, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14789, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14788, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14787, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3450", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462" + ] + }, + { + "lwp": 14786, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3450", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462" + ] + }, + { + "lwp": 14785, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3450", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462" + ] + }, + { + "lwp": 14784, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14783, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462", + "main.yielder+0 in /home/ubuntu/ebpf/tools/coredump/testsources/go/mcallstop/main.go:14", + "runtime.goexit+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:1694" + ] + }, + { + "lwp": 14782, + "frames": [ + "runtime.procyield+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:806", + "runtime.lock2+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_spinbit.go:216", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3450", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4306", + "runtime.mcall+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:462" + ] + }, + { + "lwp": 14781, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 14780, + "frames": [ + "runtime.usleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:135", + "runtime.sysmon+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:6234", + "runtime.sysmon+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:6234", + "runtime.mstart1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1928", + "runtime.mstart0+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1890", + "runtime.mstart+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:396" + ] + } + ], + "modules": [ + { + "ref": "66360a0baedbe71fc18f3225c749bde80c7d1baa2e9f77086f0e12c496c383ee", + "local-path": "/home/ubuntu/ebpf/test-mcallstop" + } + ] +} diff --git a/tools/coredump/testdata/amd64/go-systemstack.json b/tools/coredump/testdata/amd64/go-systemstack.json new file mode 100644 index 000000000..b8ed10362 --- /dev/null +++ b/tools/coredump/testdata/amd64/go-systemstack.json @@ -0,0 +1,75 @@ +{ + "coredump-ref": "ee9b687ee6e4537ceaf3eb330e0133b1168a6133a824d49a89171f2554a4243c", + "threads": [ + { + "lwp": 17092, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.newstack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/stack.go:1130", + "runtime.morestack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:621", + "runtime.newproc+0 in :1", + "" + ] + }, + { + "lwp": 17096, + "frames": [ + "runtime.futex+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:558", + "runtime.futexsleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/os_linux.go:76", + "runtime.notesleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3374", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.newstack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/stack.go:1130", + "runtime.morestack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:621", + "" + ] + }, + { + "lwp": 17095, + "frames": [ + "linux-vdso.1.so+0xcfd", + "runtime.nanotime1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:265", + "" + ] + }, + { + "lwp": 17094, + "frames": [ + "internal/runtime/syscall.Syscall6+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/internal/runtime/syscall/asm_linux_amd64.s:36", + "internal/runtime/syscall.EpollWait+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/internal/runtime/syscall/syscall_linux.go:34", + "runtime.netpoll+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/netpoll_epoll.go:120", + "runtime.findRunnable+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:3728", + "runtime.schedule+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:4301", + "runtime.newstack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/stack.go:1130", + "runtime.morestack+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:621", + "" + ] + }, + { + "lwp": 17093, + "frames": [ + "runtime.usleep+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/sys_linux_amd64.s:135", + "runtime.sysmon+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:6234", + "runtime.sysmon+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:6234", + "runtime.mstart1+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1928", + "runtime.mstart0+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/proc.go:1890", + "runtime.mstart+0 in /home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.0.linux-amd64/src/runtime/asm_amd64.s:396" + ] + } + ], + "modules": [ + { + "ref": "af87dae1fa2f9e4863c87f3c77d1405f1578f5ace132fdbe51d221a35ab08ac9", + "local-path": "/home/ubuntu/ebpf/test-systemstack" + } + ] +} diff --git a/tools/coredump/testdata/arm64/go-mcallpark.json b/tools/coredump/testdata/arm64/go-mcallpark.json new file mode 100644 index 000000000..5bceafaa3 --- /dev/null +++ b/tools/coredump/testdata/arm64/go-mcallpark.json @@ -0,0 +1,104 @@ +{ + "coredump-ref": "0a4c90cc40055029402c1a9d9f4f57aeb9cb4f4c2c6835d96d527a73d14ca96f", + "threads": [ + { + "lwp": 22411, + "frames": [ + "internal/runtime/syscall.Syscall6+0 in /usr/local/go/src/internal/runtime/syscall/asm_linux_arm64.s:17", + "runtime.netpoll+0 in /usr/local/go/src/runtime/netpoll_epoll.go:120", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3728", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "time.Sleep+0 in /usr/local/go/src/runtime/time.go:365", + "main.main+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallpark/main.go:27", + "runtime.main+0 in /usr/local/go/src/internal/runtime/atomic/types.go:194", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22422, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3374", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.mstart1+0 in /usr/local/go/src/runtime/proc.go:1936", + "runtime.mstart0+0 in /usr/local/go/src/runtime/proc.go:1858", + "runtime.mstart+0 in /usr/local/go/src/runtime/asm_arm64.s:180" + ] + }, + { + "lwp": 22421, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3374", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /usr/local/go/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /usr/local/go/src/runtime/chan.go:510", + "main.parker+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22420, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3374", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.chanrecv+0 in /usr/local/go/src/runtime/chan.go:670", + "runtime.chanrecv1+0 in /usr/local/go/src/runtime/chan.go:510", + "main.parker+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallpark/main.go:16", + "main.main.gowrap1+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallpark/main.go:24", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22419, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.startlockedm+0 in /usr/local/go/src/runtime/proc.go:3283", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4128", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.bgsweep+0 in /usr/local/go/src/runtime/mgcsweep.go:301", + "runtime.gcenable.gowrap1+0 in /usr/local/go/src/runtime/mgc.go:212", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22418, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notetsleep_internal+0 in /usr/local/go/src/runtime/lock_futex.go:90", + "runtime.notetsleep+0 in /usr/local/go/src/runtime/lock_futex.go:112", + "runtime.sysmon+0 in /usr/local/go/src/runtime/proc.go:6270", + "runtime.mstart1+0 in /usr/local/go/src/runtime/proc.go:1931", + "runtime.mstart0+0 in /usr/local/go/src/runtime/proc.go:1858", + "runtime.mstart+0 in /usr/local/go/src/runtime/asm_arm64.s:180" + ] + } + ], + "modules": [ + { + "ref": "a356938f05d9d82350a4dd0d973b754422c6bcefca2934858c06abd714c0fbb7", + "local-path": "/home/tintin/ebpf/test-mcallpark" + } + ] +} diff --git a/tools/coredump/testdata/arm64/go-mcallstop.json b/tools/coredump/testdata/arm64/go-mcallstop.json new file mode 100644 index 000000000..c2442fa10 --- /dev/null +++ b/tools/coredump/testdata/arm64/go-mcallstop.json @@ -0,0 +1,215 @@ +{ + "coredump-ref": "5247300db828f04354d5ac16c3bff5c4d51ef465671dba971a78d468ca737139", + "threads": [ + { + "lwp": 22277, + "frames": [ + "runtime.osyield+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:739", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.block+0 in /usr/local/go/src/runtime/select.go:105", + "main.main+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:27", + "runtime.main+0 in /usr/local/go/src/internal/runtime/atomic/types.go:194", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22294, + "frames": [ + "runtime.(*timers).check+0 in /usr/local/go/src/runtime/time.go:1011", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/traceruntime.go:151", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22293, + "frames": [ + "linux-vdso.1.so+0x320", + "runtime.nanotime1+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:364", + "runtime.casgstatus+0 in /usr/local/go/src/runtime/proc.go:1323", + "runtime.execute+0 in /usr/local/go/src/runtime/proc.go:3337", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4193", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "main.yielder+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:18", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22292, + "frames": [ + "linux-vdso.1.so+0x320", + "runtime.nanotime1+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:364", + "runtime.casgstatus+0 in /usr/local/go/src/runtime/proc.go:1323", + "runtime.execute+0 in /usr/local/go/src/runtime/proc.go:3337", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4193", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "main.yielder+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:18", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22291, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:652", + "runtime.semasleep+0 in /usr/local/go/src/runtime/lock_futex.go:147", + "runtime.lock2+0 in /usr/local/go/src/runtime/lock_spinbit.go:251", + "runtime.wakep+0 in /usr/local/go/src/runtime/proc.go:3219", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4300", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22290, + "frames": [ + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4300", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22289, + "frames": [ + "runtime.execute+0 in /usr/local/go/src/runtime/proc.go:3337", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4193", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "main.yielder+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:18", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22288, + "frames": [ + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3422", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22287, + "frames": [ + "runtime.osyield+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:739", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22286, + "frames": [ + "linux-vdso.1.so+0x360", + "runtime.nanotime1+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:364", + "runtime.(*mLockProfile).start+0 in /usr/local/go/src/runtime/time_nofake.go:33", + "runtime.lock2+0 in /usr/local/go/src/runtime/lock_spinbit.go:247", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22285, + "frames": [ + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4270", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "main.yielder+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:18", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22284, + "frames": [ + "runtime.osyield+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:739", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22283, + "frames": [ + "runtime.osyield+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:739", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3450", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 22282, + "frames": [ + "linux-vdso.1.so+0x360", + "runtime.nanotime1+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:364", + "runtime.casgstatus+0 in /usr/local/go/src/runtime/proc.go:1323", + "runtime.execute+0 in /usr/local/go/src/runtime/proc.go:3337", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4193", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "main.yielder+0 in /home/tintin/ebpf/tools/coredump/testsources/go/mcallstop/main.go:18", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22281, + "frames": [ + "runtime.lock2+0 in /usr/local/go/src/runtime/lock_spinbit.go:199", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3450", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.bgsweep+0 in /usr/local/go/src/runtime/mgcsweep.go:301", + "runtime.gcenable.gowrap1+0 in /usr/local/go/src/runtime/mgc.go:212", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22280, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:652", + "runtime.semasleep+0 in /usr/local/go/src/runtime/lock_futex.go:147", + "runtime.lock2+0 in /usr/local/go/src/runtime/lock_spinbit.go:251", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4293", + "runtime.gosched_m+0 in /usr/local/go/src/runtime/proc.go:4306", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242", + "runtime.gopark+0 in /usr/local/go/src/runtime/proc.go:461", + "runtime.(*scavengerState).park+0 in /usr/local/go/src/runtime/mgcscavenge.go:426", + "runtime.bgscavenge+0 in /usr/local/go/src/runtime/mgcscavenge.go:656", + "runtime.gcenable.gowrap2+0 in /usr/local/go/src/runtime/mgc.go:213", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 22279, + "frames": [ + "runtime.usleep+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:138", + "runtime.sysmon+0 in /usr/local/go/src/runtime/proc.go:6251", + "runtime.mstart1+0 in /usr/local/go/src/runtime/proc.go:1931", + "runtime.mstart0+0 in /usr/local/go/src/runtime/proc.go:1858", + "runtime.mstart+0 in /usr/local/go/src/runtime/asm_arm64.s:180" + ] + } + ], + "modules": [ + { + "ref": "3bc9854ee9afddcf89db17726c9afcc385dbe19225d3406ca718e06b54a10f77", + "local-path": "/home/tintin/ebpf/test-mcallstop" + } + ] +} diff --git a/tools/coredump/testdata/arm64/go-systemstack.json b/tools/coredump/testdata/arm64/go-systemstack.json new file mode 100644 index 000000000..edb768265 --- /dev/null +++ b/tools/coredump/testdata/arm64/go-systemstack.json @@ -0,0 +1,85 @@ +{ + "coredump-ref": "12aeb71e72a7856db3371aab97d522d8abe9c68618322716632afb692b58fbe2", + "threads": [ + { + "lwp": 9982, + "frames": [ + "runtime.(*unwinder).symPC+0 in /usr/local/go/src/runtime/traceback.go:581", + "runtime.tracebackPCs+0 in /usr/local/go/src/runtime/traceback.go:628", + "runtime.callers.func1+0 in /usr/local/go/src/runtime/traceback.go:1100", + "runtime.systemstack+0 in /usr/local/go/src/runtime/asm_arm64.s:297", + "runtime.callers+0 in /usr/local/go/src/runtime/traceback.go:1102", + "main.main+0 in /home/tintin/ebpf/tools/coredump/testsources/go/systemstack/main.go:33", + "runtime.main+0 in /usr/local/go/src/internal/runtime/atomic/types.go:194", + "runtime.goexit+0 in /usr/local/go/src/runtime/asm_arm64.s:1269" + ] + }, + { + "lwp": 9993, + "frames": [ + "internal/runtime/syscall.Syscall6+0 in /usr/local/go/src/internal/runtime/syscall/asm_linux_arm64.s:17", + "runtime.netpoll+0 in /usr/local/go/src/runtime/netpoll_epoll.go:120", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3728", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 9992, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3374", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.newstack+0 in /usr/local/go/src/runtime/stack.go:1130", + "runtime.morestack+0 in /usr/local/go/src/runtime/asm_arm64.s:396", + "" + ] + }, + { + "lwp": 9991, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.startlockedm+0 in /usr/local/go/src/runtime/proc.go:3283", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4128", + "runtime.park_m+0 in /usr/local/go/src/runtime/proc.go:4267", + "runtime.mcall+0 in /usr/local/go/src/runtime/asm_arm64.s:242" + ] + }, + { + "lwp": 9990, + "frames": [ + "runtime.futex+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:651", + "runtime.notesleep+0 in /usr/local/go/src/runtime/lock_futex.go:48", + "runtime.stopm+0 in /usr/local/go/src/runtime/proc.go:1961", + "runtime.findRunnable+0 in /usr/local/go/src/runtime/proc.go:3374", + "runtime.schedule+0 in /usr/local/go/src/runtime/proc.go:4138", + "runtime.goschedImpl+0 in /usr/local/go/src/runtime/proc.go:4301", + "runtime.newstack+0 in /usr/local/go/src/runtime/stack.go:1130", + "runtime.morestack+0 in /usr/local/go/src/runtime/asm_arm64.s:396", + "" + ] + }, + { + "lwp": 9989, + "frames": [ + "runtime.usleep+0 in /usr/local/go/src/runtime/sys_linux_arm64.s:138", + "runtime.sysmon+0 in /usr/local/go/src/runtime/proc.go:6251", + "runtime.mstart1+0 in /usr/local/go/src/runtime/proc.go:1931", + "runtime.mstart0+0 in /usr/local/go/src/runtime/proc.go:1858", + "runtime.mstart+0 in /usr/local/go/src/runtime/asm_arm64.s:180" + ] + } + ], + "modules": [ + { + "ref": "aa6217c134a0cedb8f170637641ccbc9e61f17b1fc1b137efef4653364e7d8ab", + "local-path": "/home/tintin/ebpf/test-systemstack" + } + ] +} diff --git a/tools/coredump/testsources/go/mcallpark/main.go b/tools/coredump/testsources/go/mcallpark/main.go new file mode 100644 index 000000000..0fab4dbdf --- /dev/null +++ b/tools/coredump/testsources/go/mcallpark/main.go @@ -0,0 +1,28 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "runtime" + "time" +) + +//go:noinline +func parker(ch <-chan struct{}) { + <-ch +} + +func main() { + runtime.GOMAXPROCS(8) + fmt.Println("PID:", os.Getpid()) + + ch := make(chan struct{}) + for i := 0; i < 8; i++ { + go parker(ch) + } + time.Sleep(time.Hour) +} + diff --git a/tools/coredump/testsources/go/mcallstop/main.go b/tools/coredump/testsources/go/mcallstop/main.go new file mode 100644 index 000000000..d27d4d83c --- /dev/null +++ b/tools/coredump/testsources/go/mcallstop/main.go @@ -0,0 +1,28 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Exercise the post-dropg mcall case where the candidate goroutine found from +// the g0 stack has already been rescheduled on another M. In that state, stopping +// at runtime.mcall is safer than reading a potentially stale gobuf. +package main + +import ( + "fmt" + "os" + "runtime" +) + +//go:noinline +func yielder() { + for { + runtime.Gosched() + } +} + +func main() { + fmt.Println("PID:", os.Getpid()) + for i := 0; i < 16; i++ { + go yielder() + } + select {} +} diff --git a/tools/coredump/testsources/go/systemstack/main.go b/tools/coredump/testsources/go/systemstack/main.go new file mode 100644 index 000000000..498f2b194 --- /dev/null +++ b/tools/coredump/testsources/go/systemstack/main.go @@ -0,0 +1,33 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "runtime" + "runtime/pprof" + "time" +) + +func main() { + f, err := os.Create("cpu.pprof") + if err != nil { + panic(err) + } + defer f.Close() + + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + + pcs := make([]uintptr, 32) + deadline := time.After(30 * time.Second) + for { + select { + case <-deadline: + return + default: + runtime.Callers(0, pcs) + } + } +} diff --git a/tools/errors-codegen/errors.json b/tools/errors-codegen/errors.json index e9850c747..d5eb4dfc7 100644 --- a/tools/errors-codegen/errors.json +++ b/tools/errors-codegen/errors.json @@ -364,5 +364,25 @@ "id": 7006, "name": "beam_range_search_exhausted", "description": "BEAM: Ran out of iterations searching for the current code header" + }, + { + "id": 8000, + "name": "go_mcall_no_go_offsets", + "description": "Go: No Go process info available for mcall unwinding" + }, + { + "id": 8001, + "name": "go_mcall_resolve_goroutine", + "description": "Go: Failed to resolve the user goroutine during mcall unwinding" + }, + { + "id": 8002, + "name": "go_mcall_read_gobuf", + "description": "Go: Failed to read gobuf fields from the goroutine during mcall unwinding" + }, + { + "id": 8003, + "name": "go_mcall_gobuf_not_populated", + "description": "Go: Gobuf sp/pc is zero during mcall unwinding (sched not yet populated)" } ] diff --git a/tools/gooffsets/main.go b/tools/gooffsets/main.go index a327e5c96..d57b2b2a1 100644 --- a/tools/gooffsets/main.go +++ b/tools/gooffsets/main.go @@ -19,6 +19,9 @@ type goLabelsOffsets struct { hmapCount uint32 hmapLog2BucketCount uint32 hmapBuckets uint32 + schedSp uint32 + schedPcOff uint8 + schedBpOff uint8 } func getOffsets(f *elf.File, version string) (*goLabelsOffsets, error) { @@ -35,6 +38,13 @@ func getOffsets(f *elf.File, version string) (*goLabelsOffsets, error) { if g == nil { return nil, errors.New("type runtime.g not found") } + // ReadChildTypeAndOffset repositions the reader to the field's type entry, so + // we re-seek to g.Offset before each sequential read from g's children. + r.Seek(g.Offset) + _, err = r.Next() + if err != nil { + return nil, err + } mPType, mOffset, err := ReadChildTypeAndOffset(r, "m") if err != nil { return nil, err @@ -43,16 +53,52 @@ func getOffsets(f *elf.File, version string) (*goLabelsOffsets, error) { return nil, errors.New("type of m in runtime.g is not a pointer") } - mType, err := ReadType(r, mPType) + // Read g.sched.sp and g.sched.pc: sched is a gobuf struct embedded in g. + r.Seek(g.Offset) + _, err = r.Next() if err != nil { return nil, err } - - r.Seek(mType.Offset) + schedType, schedOffset, err := ReadChildTypeAndOffset(r, "sched") + if err != nil { + return nil, err + } + r.Seek(schedType.Offset) + _, err = r.Next() + if err != nil { + return nil, err + } + _, schedSpOff, err := ReadChildTypeAndOffset(r, "sp") + if err != nil { + return nil, err + } + r.Seek(schedType.Offset) _, err = r.Next() if err != nil { return nil, err } + _, schedPcOff, err := ReadChildTypeAndOffset(r, "pc") + if err != nil { + return nil, err + } + r.Seek(schedType.Offset) + _, err = r.Next() + if err != nil { + return nil, err + } + _, schedBpOff, err := ReadChildTypeAndOffset(r, "bp") + if err != nil { + return nil, err + } + r.Seek(schedType.Offset) + _, err = r.Next() + if err != nil { + return nil, err + } + mType, err := ReadType(r, mPType) + if err != nil { + return nil, err + } r.Seek(mType.Offset) _, err = r.Next() @@ -83,9 +129,12 @@ func getOffsets(f *elf.File, version string) (*goLabelsOffsets, error) { if semver.Compare(version, "v1.24.0") >= 0 { return &goLabelsOffsets{ - mOffset: uint32(mOffset), - curg: uint32(curgOffset), - labels: uint32(labelsOffset), + mOffset: uint32(mOffset), + curg: uint32(curgOffset), + labels: uint32(labelsOffset), + schedSp: uint32(schedOffset + schedSpOff), + schedPcOff: uint8(schedPcOff), + schedBpOff: uint8(schedBpOff), }, nil } @@ -119,6 +168,9 @@ func getOffsets(f *elf.File, version string) (*goLabelsOffsets, error) { hmapCount: uint32(countOffset), hmapLog2BucketCount: uint32(bOffset), hmapBuckets: uint32(bucketsOffset), + schedSp: uint32(schedOffset + schedSpOff), + schedPcOff: uint8(schedPcOff - schedSpOff), + schedBpOff: uint8(schedBpOff - schedSpOff), }, nil } @@ -163,5 +215,8 @@ func main() { fmt.Printf("\thmap_count: %d,\n", offs.hmapCount) fmt.Printf("\thmap_log2_bucket_count: %d,\n", offs.hmapLog2BucketCount) fmt.Printf("\thmap_buckets: %d,\n", offs.hmapBuckets) + fmt.Printf("\tsched_sp: %d,\n", offs.schedSp) + fmt.Printf("\tsched_pc_off: %d,\n", offs.schedPcOff) + fmt.Printf("\tsched_bp_off: %d,\n", offs.schedBpOff) fmt.Println("},") }