Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions internal/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,11 @@ func (c *Controller) Start(ctx context.Context) error {
// So if you change this log line update also the system test.
log.Info("Attached sched monitor")

if err := trc.AttachPrctlMonitor(); err != nil {
return fmt.Errorf("failed to attach prctl monitor: %w", err)
Comment on lines +165 to +166

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Do not fail startup when the prctl tracepoint is absent

On kernels where the syscalls/sys_exit_prctl tracepoint is unavailable (for example syscall tracepoints are not enabled), this new hard error causes Controller.Start to abort the whole profiler even though late OTEL_CTX discovery is an optional enhancement. Treating the prctl monitor attach as best-effort would preserve existing profiling behavior on those kernels instead of turning this feature into a startup dependency.

Useful? React with 👍 / 👎.

}
log.Info("Attached prctl monitor")

if err := c.startTraceHandling(ctx, trc); err != nil {
return fmt.Errorf("failed to start trace handling: %w", err)
}
Expand Down
5 changes: 4 additions & 1 deletion metrics/ids.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions metrics/metrics.json
Original file line number Diff line number Diff line change
Expand Up @@ -2153,5 +2153,12 @@
"name": "GoLabelsDroppedInvalidValue",
"field": "agent.golabels.dropped.invalid_value",
"id": 294
},
{
"description": "Number of prctl(PR_SET_VMA) calls naming an anonymous mapping OTEL_CTX",
"type": "counter",
"name": "NumPrctlSetVmaOtelCtx",
"field": "bpf.num_prctl_set_vma_otel_ctx",
"id": 295
}
]
12 changes: 12 additions & 0 deletions processcontext/integrationtests/processcontext_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,16 @@ func Test_ProcessContext(t *testing.T) {
tests := map[string]struct {
exeName string
args []string
env []string
}{
"glibc_exe": {exeName: "processctx_exe_glibc"},
// Publishes the process context after a delay, so the profiler discovers
// the PID before the publication and the prctl monitor must trigger a
// resync to pick up the OTEL_CTX mapping.
"glibc_exe_delayed_publish": {
exeName: "processctx_exe_glibc",
env: []string{"OTEL_PROCESS_CTX_PUBLISH_DELAY_MS=200"},
},
// "musl_exe": {exeName: "processctx_exe_musl"},
// "glibc_lib": {exeName: "processctx_lib_glibc"},
// "musl_lib": {exeName: "processctx_lib_musl"},
Expand Down Expand Up @@ -101,12 +109,16 @@ func Test_ProcessContext(t *testing.T) {
t.Log("Attached tracer program")
require.NoError(t, trc.EnableProfiling())
require.NoError(t, trc.AttachSchedMonitor())
require.NoError(t, trc.AttachPrctlMonitor())

traceCh := make(chan *libpf.EbpfTrace)
require.NoError(t, trc.StartMapMonitors(ctx, traceCh))

cmd := exec.CommandContext(ctx, filepath.Join(exeDir, tc.exeName), tc.args...)
cmd.Stderr = os.Stderr
if len(tc.env) > 0 {
cmd.Env = append(os.Environ(), tc.env...)
}
require.NoError(t, cmd.Start())

wg := sync.WaitGroup{}
Expand Down
14 changes: 14 additions & 0 deletions processcontext/integrationtests/testdata/processctx.c
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#include "processctx_lib.h"
Expand Down Expand Up @@ -82,6 +83,19 @@ int main(int argc, char *argv[]) {

signal(SIGTERM, handle_sigterm);

// OTEL_PROCESS_CTX_PUBLISH_DELAY_MS lets the test exercise the prctl monitor
// path: the profiler discovers the PID first, then the publish prctl fires
// and triggers a resync. We burn CPU rather than sleep so the process stays
// on-CPU and is sampled by the profiler's perf event, which drives process
// synchronization before the context is published.
const char *delay_str = getenv("OTEL_PROCESS_CTX_PUBLISH_DELAY_MS");
if (delay_str != NULL) {
int delay_ms = atoi(delay_str);
if (delay_ms > 0) {
burn(delay_ms);
}
}

if (init_process_context()) {
fprintf(stderr, "Failed to initialize process context\n");
return 1;
Expand Down
6 changes: 2 additions & 4 deletions processmanager/processinfo.go
Original file line number Diff line number Diff line change
Expand Up @@ -557,10 +557,8 @@ func (pm *ProcessManager) SynchronizeProcess(pr process.Process) {
numParseErrors, err := pr.IterateMappings(func(m process.RawMapping) bool {
if processcontext.IsContextMapping(m.IsExecutable(), m.Path) {
contextMappingAddr = m.Vaddr
// Even if process context is not found, it might be published in the future.
// For now, we rely on a new call to synchronizeMappings to pick it up.
// TODO: Add some kind of polling mechanism or a hook on prctl to be notified
// when the process context is published.
// The eBPF hook on prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME) will trigger a
// PID resynchronization when the process names its context mapping "OTEL_CTX".
}

// Executable mappings and VDSO, converted directly to libpf.FrameMapping
Expand Down
88 changes: 88 additions & 0 deletions support/ebpf/prctl_monitor.ebpf.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// This file contains the code for the tracepoint on prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ...)
// to detect when a process names an anonymous memory mapping "OTEL_CTX".

#include "bpfdefs.h"
#include "tracemgmt.h"

#include "types.h"

#ifndef TESTING_COREDUMP

// prctl constants from include/uapi/linux/prctl.h
#define PR_SET_VMA 0x53564d41
#define PR_SET_VMA_ANON_NAME 0

// tracepoint__sys_exit_prctl detects when a process names an anonymous VMA
// "OTEL_CTX" and triggers a PID resynchronization so the profiler can discover
// the newly published process context mapping.
//
// We hook syscall exit, not entry, so the resync runs after the kernel has applied
// the rename; otherwise user space could re-read /proc/<pid>/maps before
// "[anon:OTEL_CTX]" is visible and miss the freshly published context.
//
// The exit tracepoint only carries the return value, so the prctl arguments are
// recovered from the task's entry pt_regs (preserved across the syscall).
SEC("tracepoint/syscalls/sys_exit_prctl")
int tracepoint__sys_exit_prctl(void *ctx)
{
struct task_struct *task = (struct task_struct *)bpf_get_current_task();
long ptregs_addr = get_task_pt_regs(task);
if (!ptregs_addr) {
goto exit;
}

struct pt_regs regs;
if (bpf_probe_read_kernel(&regs, sizeof(regs), (void *)ptregs_addr)) {
goto exit;
}

// prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4,
// unsigned long arg5): we only need option, arg2 and arg5 (the name).
#if defined(__x86_64__)
unsigned long option = regs.di;
unsigned long arg2 = regs.si;
unsigned long name_ptr = regs.r8;
#elif defined(__aarch64__)
// At exit x0 (regs[0]) holds the return value, arg1 is preserved in orig_x0.
unsigned long option = regs.orig_x0;
unsigned long arg2 = regs.regs[1];
unsigned long name_ptr = regs.regs[4];
#else
#error unsupported architecture
#endif

if (option != PR_SET_VMA || arg2 != PR_SET_VMA_ANON_NAME) {
goto exit;
}

u64 pid_tgid = bpf_get_current_pid_tgid();
u32 pid = pid_tgid >> 32;

if (!bpf_map_lookup_elem(&reported_pids, &pid) && !pid_information_exists(pid)) {
// Only report PIDs that we explicitly track. This avoids sending kernel worker PIDs
// to userspace.
goto exit;
}

// Read the VMA name from user-space. We only need 9 bytes ("OTEL_CTX" + NUL).
__attribute__((aligned(8))) char name[9] = {};
if (bpf_probe_read_user(name, sizeof(name), (void *)name_ptr)) {
goto exit;
}

// Check for an exact "OTEL_CTX" match. We avoid bpf_strncmp (kernel 5.17+).
// Instead, compare as a u64 for the 8 characters plus a byte check for the
// NUL terminator.
if (*(u64 *)name != *(u64 *)"OTEL_CTX" || name[8] != '\0') {
goto exit;
}

if (report_pid(ctx, pid_tgid, RATELIMIT_ACTION_DEFAULT)) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Bypass rate limiting for OTEL_CTX publication

When an already-tracked process has a recent reported_pids token from another PID event (for example an unknown-PC resync), RATELIMIT_ACTION_DEFAULT can suppress this one-shot prctl notification even though the earlier synchronization may have run before the mapping was named. Since SynchronizeProcess only removes reported_pids for new processes (processmanager/processinfo.go notes non-new syncs remain rate-limited), that leaves no guaranteed later resync and the newly published process context can remain stale/missing; the prctl publication event should bypass/reset the PID-event rate limit.

Useful? React with 👍 / 👎.

increment_metric(metricID_NumPrctlSetVmaOtelCtx);
}

exit:
return 0;
}

#endif
Binary file modified support/ebpf/tracer.ebpf.amd64
Binary file not shown.
Binary file modified support/ebpf/tracer.ebpf.arm64
Binary file not shown.
3 changes: 3 additions & 0 deletions support/ebpf/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,9 @@ enum {
// number of bpf_ringbuf_output failures
metricID_BPFRingbufOutputErr,

// number of prctl(PR_SET_VMA) calls naming an anonymous mapping OTEL_CTX
metricID_NumPrctlSetVmaOtelCtx,

//
// Metric IDs above are for counters (cumulative values)
//
Expand Down
3 changes: 2 additions & 1 deletion support/types.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions support/types_def.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,4 +309,5 @@ var MetricsTranslation = []metrics.MetricID{
C.metricID_UnwindRubyErrCmeMaxEp: metrics.IDUnwindRubyErrCmeMaxEp,
C.metricID_UnwindErrBadDTVRead: metrics.IDUnwindErrBadDTVRead,
C.metricID_BPFRingbufOutputErr: metrics.IDBPFRingbufOutputErr,
C.metricID_NumPrctlSetVmaOtelCtx: metrics.IDNumPrctlSetVmaOtelCtx,
}
12 changes: 12 additions & 0 deletions tracer/tracepoints.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,15 @@ func (t *Tracer) AttachSchedMonitor() error {
name := schedProcessFreeHookName(libpf.MapKeysToSet(t.ebpfProgs))
return t.attachToTracepoint("sched", "sched_process_free", t.ebpfProgs[name])
}

// AttachPrctlMonitor attaches a tracepoint on prctl() to detect when a process
// names an anonymous VMA "OTEL_CTX" via prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ...).
// This triggers a PID resynchronization so the profiler can discover newly published
// process context mappings.
func (t *Tracer) AttachPrctlMonitor() error {
prog, ok := t.ebpfProgs["tracepoint__sys_exit_prctl"]
if !ok {
return fmt.Errorf("eBPF program tracepoint__sys_exit_prctl not found")
}
return t.attachToTracepoint("syscalls", "sys_exit_prctl", prog)
}
7 changes: 6 additions & 1 deletion tracer/tracer.go
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,7 @@ func loadPerfUnwinders(coll *cebpf.CollectionSpec, ebpfProgs map[string]*cebpf.P
LogLevel: cebpf.LogLevel(bpfVerifierLogLevel),
}

progs := make([]progLoaderHelper, len(tailCallProgs)+2)
progs := make([]progLoaderHelper, len(tailCallProgs)+3)
copy(progs, tailCallProgs)

schedProcessFree := schedProcessFreeHookName(libpf.MapKeysToSet(coll.Programs))
Expand All @@ -741,6 +741,11 @@ func loadPerfUnwinders(coll *cebpf.CollectionSpec, ebpfProgs map[string]*cebpf.P
noTailCallTarget: true,
enable: true,
},
progLoaderHelper{
name: "tracepoint__sys_exit_prctl",
noTailCallTarget: true,
enable: true,
},
progLoaderHelper{
name: "native_tracer_entry",
noTailCallTarget: true,
Expand Down
Loading