Skip to content

Commit 1ce0b30

Browse files
authored
fix(crashtracking): multi thread collection centos flakes harden (#2113)
[PROF-15089](https://datadoghq.atlassian.net/jira/software/c/projects/PROF/boards/11?selectedIssue=PROF-15089) # What does this PR do? On older kernels, there may be a race where `waitpid` reports a thread as stopped but the register state hasn't been flushed to the ptrace-accessible area yet. Reading registers in that window gives zeros, causing libunwind to produce empty stack traces. Adds a `wait_for_registers` polling loop after `attach_thread` that spins on `PTRACE_PEEKUSER` until the instruction pointer is non-zero or the existing deadline expires. On modern kernels this should return immediately on the first iteration. # Motivation More [flakes](https://app.datadoghq.com/ci/pipeline-executions?query=ci_level%3Apipeline%20%40git.repository.name%3ADataDog%2A%2Flibdatadog%20%28%40git.is_default_branch%3Atrue%20OR%20%40git.branch%3Amq%2A%29%20%40ci.status%3Aerror%20%21%40ci.pipeline.name%3ADDCI%2A%20%21%40ci.pipeline.name%3A%22Release%20-%20Open%20a%20release%20proposal%20PR%22&agg_m=count&agg_m_source=base&agg_t=count&fromUser=false&index=cipipeline&refresh_mode=sliding&tab=logs&start=1780928483247&end=1781533283247&paused=false) on multi thread collection tests # Additional Notes Anything else we should know when reviewing? # How to test the change? I ran all thread collection tests on cent os 10x and observed no failures [PROF-15089]: https://datadoghq.atlassian.net/browse/PROF-15089?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
1 parent c79d783 commit 1ce0b30

1 file changed

Lines changed: 38 additions & 0 deletions

File tree

libdd-crashtracker/src/receiver/ptrace_collector.rs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,12 @@ fn wait_for_stop(tid: libc::pid_t, deadline: Instant) -> Result<(), PtraceError>
129129
/// to enter ptrace-stop state before returning.
130130
///
131131
/// `stop_deadline` bounds how long we poll for the stop event.
132+
///
133+
/// After the thread enters ptrace-stop, this function also polls until the
134+
/// instruction pointer is non-zero. On older kernels there may be a race where
135+
/// `waitpid` returns WIFSTOPPED but the thread's register state hasn't been fully
136+
/// flushed to the ptrace-accessible area yet. Reading registers in that window
137+
/// yields zeros, which causes libunwind to produce an empty stack trace.
132138
fn attach_thread(tid: libc::pid_t, stop_deadline: Instant) -> Result<(), PtraceError> {
133139
// PTRACE_SEIZE attaches without stopping the thread
134140
let result = unsafe {
@@ -163,9 +169,41 @@ fn attach_thread(tid: libc::pid_t, stop_deadline: Instant) -> Result<(), PtraceE
163169
let _ = detach_thread(tid);
164170
return Err(e);
165171
}
172+
173+
// On older kernels, the register state may not be
174+
// immediately readable after waitpid reports the stop. Spin briefly
175+
// until PEEKUSER returns a non-zero IP, proving registers are committed.
176+
wait_for_registers(tid, stop_deadline);
177+
166178
Ok(())
167179
}
168180

181+
/// Poll the thread's instruction pointer using PTRACE_PEEKUSER until it is
182+
/// non-zero or the deadline expires. On modern kernels this should return on the
183+
/// first iteration; on older ones, it may take a few microseconds.
184+
fn wait_for_registers(tid: libc::pid_t, deadline: Instant) {
185+
#[cfg(target_arch = "x86_64")]
186+
const IP_OFFSET: libc::c_long = 16 * std::mem::size_of::<libc::c_long>() as libc::c_long; // RIP
187+
188+
#[cfg(target_arch = "aarch64")]
189+
const IP_OFFSET: libc::c_long = 32 * std::mem::size_of::<libc::c_long>() as libc::c_long; // PC
190+
191+
const SPIN_SLEEP: Duration = Duration::from_micros(100);
192+
193+
loop {
194+
unsafe { *libc::__errno_location() = 0 };
195+
let ip = unsafe { libc::ptrace(libc::PTRACE_PEEKUSER, tid as libc::c_long, IP_OFFSET, 0) };
196+
// PEEKUSER returns the register value; errno==0 means success.
197+
if ip != 0 && unsafe { *libc::__errno_location() } == 0 {
198+
return;
199+
}
200+
if Instant::now() >= deadline {
201+
return;
202+
}
203+
std::thread::sleep(SPIN_SLEEP);
204+
}
205+
}
206+
169207
fn detach_thread(tid: libc::pid_t) -> Result<(), PtraceError> {
170208
// SAFETY: PTRACE_DETACH is valid for a currently-traced thread
171209
let result = unsafe {

0 commit comments

Comments
 (0)