From d6f07c705cbfc5bb027fb19189a237ece87dfbc8 Mon Sep 17 00:00:00 2001 From: "gyuheon.oh" Date: Wed, 8 Jul 2026 21:56:42 +0000 Subject: [PATCH 1/6] Harden ptrace retries --- .../src/receiver/ptrace_collector.rs | 85 +++++++++++++++---- 1 file changed, 68 insertions(+), 17 deletions(-) diff --git a/libdd-crashtracker/src/receiver/ptrace_collector.rs b/libdd-crashtracker/src/receiver/ptrace_collector.rs index 3857541c54..4d11acec73 100644 --- a/libdd-crashtracker/src/receiver/ptrace_collector.rs +++ b/libdd-crashtracker/src/receiver/ptrace_collector.rs @@ -173,7 +173,10 @@ fn attach_thread(tid: libc::pid_t, stop_deadline: Instant) -> Result<(), PtraceE // On older kernels, the register state may not be // immediately readable after waitpid reports the stop. Spin briefly // until PEEKUSER returns a non-zero IP, proving registers are committed. - wait_for_registers(tid, stop_deadline); + if !wait_for_registers(tid, stop_deadline) { + let _ = detach_thread(tid); + return Err(PtraceError::Attach(tid, libc::ETIMEDOUT)); + } Ok(()) } @@ -181,7 +184,11 @@ fn attach_thread(tid: libc::pid_t, stop_deadline: Instant) -> Result<(), PtraceE /// Poll the thread's instruction pointer using PTRACE_PEEKUSER until it is /// non-zero or the deadline expires. On modern kernels this should return on the /// first iteration; on older ones, it may take a few microseconds. -fn wait_for_registers(tid: libc::pid_t, deadline: Instant) { +/// +/// Returns `true` if a non-zero IP was observed or the check is not applicable +/// (PTRACE_PEEKUSER unsupported on the architecture), `false` if the +/// deadline expired without reading a valid IP on a platform that supports it. +fn wait_for_registers(tid: libc::pid_t, deadline: Instant) -> bool { #[cfg(target_arch = "x86_64")] const IP_OFFSET: libc::c_long = 16 * std::mem::size_of::() as libc::c_long; // RIP @@ -190,17 +197,33 @@ fn wait_for_registers(tid: libc::pid_t, deadline: Instant) { const SPIN_SLEEP: Duration = Duration::from_micros(100); + // First probe: if PTRACE_PEEKUSER returns EIO, the kernel doesn't support it + // In that case, skip the check. libunwind uses PTRACE_GETREGSET which works + // regardless, and modern kernels commit register state synchronously on ptrace-stop. + unsafe { *libc::__errno_location() = 0 }; + let ip = unsafe { libc::ptrace(libc::PTRACE_PEEKUSER, tid as libc::c_long, IP_OFFSET, 0) }; + let errno = unsafe { *libc::__errno_location() }; + if errno == libc::EIO { + return true; + } + if ip != 0 && errno == 0 { + return true; + } + loop { + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(SPIN_SLEEP); unsafe { *libc::__errno_location() = 0 }; let ip = unsafe { libc::ptrace(libc::PTRACE_PEEKUSER, tid as libc::c_long, IP_OFFSET, 0) }; - // PEEKUSER returns the register value; errno==0 means success. - if ip != 0 && unsafe { *libc::__errno_location() } == 0 { - return; + let errno = unsafe { *libc::__errno_location() }; + if errno == libc::EIO { + return true; } - if Instant::now() >= deadline { - return; + if ip != 0 && errno == 0 { + return true; } - std::thread::sleep(SPIN_SLEEP); } } @@ -369,6 +392,40 @@ pub fn capture_thread_context( /// Maximum time to wait for a single thread to enter ptrace-stop. const STOP_TIMEOUT_PER_THREAD: Duration = Duration::from_millis(50); +/// Number of retry attempts for a failed capture_thread_context call. +const CAPTURE_RETRIES: u32 = 1; + +/// Delay between retry attempts. +const RETRY_DELAY: Duration = Duration::from_millis(5); + +/// Attempt to capture a thread context with retries on transient failures. +/// +/// On some kernels, ptrace operations can fail intermittently due to scheduling +/// races or brief permission windows. Retrying once after a short delay resolves +/// most transient issues without significantly impacting overall collection time. +fn capture_with_retry( + tid: libc::pid_t, + resolve_frames: crate::StacktraceCollection, + addr_space: UnwAddrSpaceT, + overall_deadline: Instant, +) -> Option { + for attempt in 0..=CAPTURE_RETRIES { + let now = Instant::now(); + if now >= overall_deadline { + break; + } + let stop_deadline = (now + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); + match capture_thread_context(tid, resolve_frames, addr_space, stop_deadline) { + Ok(ctx) => return Some(ctx), + Err(_) if attempt < CAPTURE_RETRIES => { + std::thread::sleep(RETRY_DELAY); + } + Err(_) => break, + } + } + None +} + /// Stream thread contexts to a callback one at a time. /// /// For each thread in the process the callback receives the TID and an optional @@ -412,9 +469,8 @@ where // Process the crashing thread first so it is never dropped by the cap. if crashing_tid != 0 && tids.contains(&crashing_tid) { - let stop_deadline = (Instant::now() + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); let context = - capture_thread_context(crashing_tid, resolve_frames, addr_space, stop_deadline).ok(); + capture_with_retry(crashing_tid, resolve_frames, addr_space, overall_deadline); callback(crashing_tid, context.as_ref()); processed += 1; } @@ -423,16 +479,11 @@ where if tid == crashing_tid { continue; } - let now = Instant::now(); - if now >= overall_deadline || processed >= max_threads { + if Instant::now() >= overall_deadline || processed >= max_threads { break; } - // Cap the per-thread stop wait at STOP_TIMEOUT_PER_THREAD but never - // past the overall deadline, so one thread can't consume the budget. - let stop_deadline = (now + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); - - let context = capture_thread_context(tid, resolve_frames, addr_space, stop_deadline).ok(); + let context = capture_with_retry(tid, resolve_frames, addr_space, overall_deadline); callback(tid, context.as_ref()); processed += 1; } From 07507f69aa4be51149f4bb3d53be97a62c78628c Mon Sep 17 00:00:00 2001 From: "gyuheon.oh" Date: Thu, 9 Jul 2026 15:02:27 +0000 Subject: [PATCH 2/6] Only retry register wait timeout --- .../src/receiver/ptrace_collector.rs | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/libdd-crashtracker/src/receiver/ptrace_collector.rs b/libdd-crashtracker/src/receiver/ptrace_collector.rs index 4d11acec73..f7ac89de8e 100644 --- a/libdd-crashtracker/src/receiver/ptrace_collector.rs +++ b/libdd-crashtracker/src/receiver/ptrace_collector.rs @@ -392,38 +392,45 @@ pub fn capture_thread_context( /// Maximum time to wait for a single thread to enter ptrace-stop. const STOP_TIMEOUT_PER_THREAD: Duration = Duration::from_millis(50); -/// Number of retry attempts for a failed capture_thread_context call. -const CAPTURE_RETRIES: u32 = 1; - /// Delay between retry attempts. const RETRY_DELAY: Duration = Duration::from_millis(5); -/// Attempt to capture a thread context with retries on transient failures. +/// Returns true if err is worth retrying. +/// +/// Only ETIMEDOUT (register-wait deadline expired before IP became readable) +/// should be retired. /// -/// On some kernels, ptrace operations can fail intermittently due to scheduling -/// races or brief permission windows. Retrying once after a short delay resolves -/// most transient issues without significantly impacting overall collection time. +/// EPERM (Yama denial / missing PR_SET_PTRACER) and ESRCH (thread exited) +/// are permanent for the receiver's lifetime and are not retried. +fn is_transient_ptrace_error(err: &PtraceError) -> bool { + matches!(err, PtraceError::Attach(_, libc::ETIMEDOUT)) +} + +/// Attempt to capture a thread context, retrying once on transient failures. +/// +/// Uses a single per-thread deadline across attempts so that a slow thread +/// cannot consume more than STOP_TIMEOUT_PER_THREAD total. fn capture_with_retry( tid: libc::pid_t, resolve_frames: crate::StacktraceCollection, addr_space: UnwAddrSpaceT, overall_deadline: Instant, ) -> Option { - for attempt in 0..=CAPTURE_RETRIES { - let now = Instant::now(); - if now >= overall_deadline { - break; - } - let stop_deadline = (now + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); - match capture_thread_context(tid, resolve_frames, addr_space, stop_deadline) { - Ok(ctx) => return Some(ctx), - Err(_) if attempt < CAPTURE_RETRIES => { - std::thread::sleep(RETRY_DELAY); - } - Err(_) => break, - } + let thread_deadline = (Instant::now() + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); + + match capture_thread_context(tid, resolve_frames, addr_space, thread_deadline) { + Ok(ctx) => return Some(ctx), + Err(ref e) if is_transient_ptrace_error(e) => {} + Err(_) => return None, } - None + + let remaining = thread_deadline.saturating_duration_since(Instant::now()); + if remaining <= RETRY_DELAY { + return None; + } + std::thread::sleep(RETRY_DELAY); + + capture_thread_context(tid, resolve_frames, addr_space, thread_deadline).ok() } /// Stream thread contexts to a callback one at a time. From 4f21b6ea731d3adcaa10aad8b6e4fff11e56b558 Mon Sep 17 00:00:00 2001 From: "gyuheon.oh" Date: Thu, 9 Jul 2026 15:05:16 +0000 Subject: [PATCH 3/6] Run test 100x --- .github/workflows/test.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8673c56be6..b871802408 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -269,6 +269,7 @@ jobs: needs: setup if: needs.setup.outputs.packages-count != '0' runs-on: ubuntu-latest + timeout-minutes: 720 env: PACKAGES: ${{ needs.setup.outputs.packages }} CRASHTRACKER_FEATURE: ${{ needs.setup.outputs.crashtracker-feature }} @@ -297,7 +298,7 @@ jobs: cache-bin: true # cache the ~/.cargo/bin directory - name: Build CentOS 7 Docker image run: docker build -t libdatadog-centos7 -f tools/docker/Dockerfile.centos . - - name: "cargo nextest run (centos7)" + - name: "cargo nextest run (centos7) x100" # Run docker as a user, not the default root # Mount and use the runner's toolchain as it's the same arch # exclude tracing integration tests since they require docker in docker to run a test-agent @@ -310,17 +311,20 @@ jobs: # shellcheck disable=SC2086 NEXTEST_CMD="cargo nextest run $PACKAGES $CRASHTRACKER_FEATURE --profile ci --no-tests=pass -E '!test(tracing_integration_tests::)'" fi - docker run --rm \ - --user "$(id -u):$(id -g)" \ - -v "${{ github.workspace }}:/workspace" \ - -v "${CARGO_HOME:-$HOME/.cargo}:/usr/local/cargo" \ - -v "${RUSTUP_HOME:-$HOME/.rustup}:/usr/local/rustup" \ - -e CARGO_HOME=/usr/local/cargo \ - -e RUSTUP_HOME=/usr/local/rustup \ - -e PATH=/usr/local/cargo/bin:/usr/local/rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - -w /workspace \ - libdatadog-centos7 \ - sh -c "$NEXTEST_CMD" + for i in $(seq 1 100); do + echo "=== Run $i/100 ===" + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -v "${{ github.workspace }}:/workspace" \ + -v "${CARGO_HOME:-$HOME/.cargo}:/usr/local/cargo" \ + -v "${RUSTUP_HOME:-$HOME/.rustup}:/usr/local/rustup" \ + -e CARGO_HOME=/usr/local/cargo \ + -e RUSTUP_HOME=/usr/local/rustup \ + -e PATH=/usr/local/cargo/bin:/usr/local/rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -w /workspace \ + libdatadog-centos7 \ + sh -c "$NEXTEST_CMD" + done - name: Add file attributes to JUnit XML if: success() || failure() run: cargo run --bin add_junit_file_attributes -- target/nextest/ci/junit.xml From dc76d1ffe52c17badb2b0d9f03e6a4fce2a5a8d8 Mon Sep 17 00:00:00 2001 From: "gyuheon.oh" Date: Thu, 9 Jul 2026 17:54:20 +0000 Subject: [PATCH 4/6] Retry for 0 frames --- .../src/receiver/ptrace_collector.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/libdd-crashtracker/src/receiver/ptrace_collector.rs b/libdd-crashtracker/src/receiver/ptrace_collector.rs index f7ac89de8e..ae3c5d0e0e 100644 --- a/libdd-crashtracker/src/receiver/ptrace_collector.rs +++ b/libdd-crashtracker/src/receiver/ptrace_collector.rs @@ -410,6 +410,11 @@ fn is_transient_ptrace_error(err: &PtraceError) -> bool { /// /// Uses a single per-thread deadline across attempts so that a slow thread /// cannot consume more than STOP_TIMEOUT_PER_THREAD total. +/// +/// A capture that succeeds but produces zero frames is also retried: on a +/// running thread with a confirmed non-zero IP, empty frames indicates a +/// transient libunwind issue (e.g. stale address-space cache state) rather +/// than a permanent problem. fn capture_with_retry( tid: libc::pid_t, resolve_frames: crate::StacktraceCollection, @@ -418,10 +423,16 @@ fn capture_with_retry( ) -> Option { let thread_deadline = (Instant::now() + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); - match capture_thread_context(tid, resolve_frames, addr_space, thread_deadline) { - Ok(ctx) => return Some(ctx), - Err(ref e) if is_transient_ptrace_error(e) => {} - Err(_) => return None, + let should_retry = + match capture_thread_context(tid, resolve_frames, addr_space, thread_deadline) { + Ok(ctx) if !ctx.stack_trace.frames.is_empty() => return Some(ctx), + Ok(_) => true, + Err(ref e) if is_transient_ptrace_error(e) => true, + Err(_) => false, + }; + + if !should_retry { + return None; } let remaining = thread_deadline.saturating_duration_since(Instant::now()); From eac14140a08602fb30de0078950edb274698a76a Mon Sep 17 00:00:00 2001 From: "gyuheon.oh" Date: Thu, 9 Jul 2026 20:40:09 +0000 Subject: [PATCH 5/6] Trigger CI --- .../src/receiver/ptrace_collector.rs | 82 ++++++++++++------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/libdd-crashtracker/src/receiver/ptrace_collector.rs b/libdd-crashtracker/src/receiver/ptrace_collector.rs index ae3c5d0e0e..bb2f1cc299 100644 --- a/libdd-crashtracker/src/receiver/ptrace_collector.rs +++ b/libdd-crashtracker/src/receiver/ptrace_collector.rs @@ -173,10 +173,15 @@ fn attach_thread(tid: libc::pid_t, stop_deadline: Instant) -> Result<(), PtraceE // On older kernels, the register state may not be // immediately readable after waitpid reports the stop. Spin briefly // until PEEKUSER returns a non-zero IP, proving registers are committed. - if !wait_for_registers(tid, stop_deadline) { - let _ = detach_thread(tid); - return Err(PtraceError::Attach(tid, libc::ETIMEDOUT)); - } + // + // If the deadline expires before we see a non-zero IP, proceed anyway: + // libunwind uses PTRACE_GETREGSET which may succeed even when PEEKUSER + // returns zero. If registers are truly uncommitted, unwind_remote_thread + // will return 0 frames and capture_with_retry will retry without needing + // a costly detach/re-attach cycle (which can fail with EPERM under CPU + // pressure because the kernel hasn't fully released the prior ptrace + // state). + let _ = wait_for_registers(tid, stop_deadline); Ok(()) } @@ -245,6 +250,14 @@ fn detach_thread(tid: libc::pid_t) -> Result<(), PtraceError> { return Err(PtraceError::Detach(tid, errno)); } } + + // Drain any pending waitpid event so the kernel fully releases the thread. + // Without this, a rapid re-attach (PTRACE_SEIZE) can fail with EPERM under + // CPU pressure because the kernel hasn't finished processing the detach. + unsafe { + libc::waitpid(tid, ptr::null_mut(), libc::__WALL | libc::WNOHANG); + } + Ok(()) } @@ -390,58 +403,71 @@ pub fn capture_thread_context( } /// Maximum time to wait for a single thread to enter ptrace-stop. -const STOP_TIMEOUT_PER_THREAD: Duration = Duration::from_millis(50); +/// +/// 200ms accommodates CI environments with heavy CPU contention where the thread +/// may take tens of milliseconds to be scheduled and enter ptrace-stop after +/// PTRACE_INTERRUPT. On unloaded machines the stop typically arrives in <1ms. +const STOP_TIMEOUT_PER_THREAD: Duration = Duration::from_millis(200); -/// Delay between retry attempts. -const RETRY_DELAY: Duration = Duration::from_millis(5); +/// Delay between retry attempts. Uses an exponential back-off starting from +/// this value (10ms, 20ms, 40ms). +const RETRY_BASE_DELAY: Duration = Duration::from_millis(10); + +/// Maximum number of retry attempts per thread. +const MAX_RETRIES: u32 = 3; /// Returns true if err is worth retrying. /// -/// Only ETIMEDOUT (register-wait deadline expired before IP became readable) -/// should be retired. -/// /// EPERM (Yama denial / missing PR_SET_PTRACER) and ESRCH (thread exited) /// are permanent for the receiver's lifetime and are not retried. fn is_transient_ptrace_error(err: &PtraceError) -> bool { matches!(err, PtraceError::Attach(_, libc::ETIMEDOUT)) } -/// Attempt to capture a thread context, retrying once on transient failures. +/// Attempt to capture a thread context, retrying on transient failures. /// -/// Uses a single per-thread deadline across attempts so that a slow thread -/// cannot consume more than STOP_TIMEOUT_PER_THREAD total. +/// Each attempt gets its own `STOP_TIMEOUT_PER_THREAD` budget (capped at the +/// overall deadline) so that a retry after a timeout-induced failure actually +/// has enough time to succeed. On older kernels (e.g. CentOS 7 / kernel 3.10) +/// the first attempt can consume its entire budget waiting for registers to +/// become readable; reusing that exhausted deadline would make the retry a +/// no-op. /// /// A capture that succeeds but produces zero frames is also retried: on a /// running thread with a confirmed non-zero IP, empty frames indicates a /// transient libunwind issue (e.g. stale address-space cache state) rather -/// than a permanent problem. +/// than a permanent problem. Retries use exponential back-off to give the +/// kernel time to fully commit thread state between attempts. fn capture_with_retry( tid: libc::pid_t, resolve_frames: crate::StacktraceCollection, addr_space: UnwAddrSpaceT, overall_deadline: Instant, ) -> Option { - let thread_deadline = (Instant::now() + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); + for attempt in 0..=MAX_RETRIES { + let thread_deadline = (Instant::now() + STOP_TIMEOUT_PER_THREAD).min(overall_deadline); - let should_retry = match capture_thread_context(tid, resolve_frames, addr_space, thread_deadline) { Ok(ctx) if !ctx.stack_trace.frames.is_empty() => return Some(ctx), - Ok(_) => true, - Err(ref e) if is_transient_ptrace_error(e) => true, - Err(_) => false, - }; + Ok(_) => {} // 0 frames -- retry + Err(ref e) if is_transient_ptrace_error(e) => {} // ETIMEDOUT -- retry + Err(_) => return None, // permanent error + } - if !should_retry { - return None; - } + if attempt == MAX_RETRIES { + break; + } - let remaining = thread_deadline.saturating_duration_since(Instant::now()); - if remaining <= RETRY_DELAY { - return None; + let delay = RETRY_BASE_DELAY * 2u32.saturating_pow(attempt); + if Instant::now() + delay >= overall_deadline { + break; + } + std::thread::sleep(delay); } - std::thread::sleep(RETRY_DELAY); - capture_thread_context(tid, resolve_frames, addr_space, thread_deadline).ok() + // All attempts produced 0 frames or timed out; return None so the caller + // records the thread with an incomplete stack rather than frames: []. + None } /// Stream thread contexts to a callback one at a time. From 7ebd2b92ea83ce46c0eacd5e042fa5822bed2507 Mon Sep 17 00:00:00 2001 From: Gyuheon Oh Date: Wed, 15 Jul 2026 13:09:33 +0000 Subject: [PATCH 6/6] Revert 100x run --- .github/workflows/test.yml | 28 ++++++++----------- .../src/receiver/ptrace_collector.rs | 12 ++------ 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b871802408..8673c56be6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -269,7 +269,6 @@ jobs: needs: setup if: needs.setup.outputs.packages-count != '0' runs-on: ubuntu-latest - timeout-minutes: 720 env: PACKAGES: ${{ needs.setup.outputs.packages }} CRASHTRACKER_FEATURE: ${{ needs.setup.outputs.crashtracker-feature }} @@ -298,7 +297,7 @@ jobs: cache-bin: true # cache the ~/.cargo/bin directory - name: Build CentOS 7 Docker image run: docker build -t libdatadog-centos7 -f tools/docker/Dockerfile.centos . - - name: "cargo nextest run (centos7) x100" + - name: "cargo nextest run (centos7)" # Run docker as a user, not the default root # Mount and use the runner's toolchain as it's the same arch # exclude tracing integration tests since they require docker in docker to run a test-agent @@ -311,20 +310,17 @@ jobs: # shellcheck disable=SC2086 NEXTEST_CMD="cargo nextest run $PACKAGES $CRASHTRACKER_FEATURE --profile ci --no-tests=pass -E '!test(tracing_integration_tests::)'" fi - for i in $(seq 1 100); do - echo "=== Run $i/100 ===" - docker run --rm \ - --user "$(id -u):$(id -g)" \ - -v "${{ github.workspace }}:/workspace" \ - -v "${CARGO_HOME:-$HOME/.cargo}:/usr/local/cargo" \ - -v "${RUSTUP_HOME:-$HOME/.rustup}:/usr/local/rustup" \ - -e CARGO_HOME=/usr/local/cargo \ - -e RUSTUP_HOME=/usr/local/rustup \ - -e PATH=/usr/local/cargo/bin:/usr/local/rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - -w /workspace \ - libdatadog-centos7 \ - sh -c "$NEXTEST_CMD" - done + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -v "${{ github.workspace }}:/workspace" \ + -v "${CARGO_HOME:-$HOME/.cargo}:/usr/local/cargo" \ + -v "${RUSTUP_HOME:-$HOME/.rustup}:/usr/local/rustup" \ + -e CARGO_HOME=/usr/local/cargo \ + -e RUSTUP_HOME=/usr/local/rustup \ + -e PATH=/usr/local/cargo/bin:/usr/local/rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin:/opt/rh/devtoolset-11/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -w /workspace \ + libdatadog-centos7 \ + sh -c "$NEXTEST_CMD" - name: Add file attributes to JUnit XML if: success() || failure() run: cargo run --bin add_junit_file_attributes -- target/nextest/ci/junit.xml diff --git a/libdd-crashtracker/src/receiver/ptrace_collector.rs b/libdd-crashtracker/src/receiver/ptrace_collector.rs index bb2f1cc299..8f09e9ab32 100644 --- a/libdd-crashtracker/src/receiver/ptrace_collector.rs +++ b/libdd-crashtracker/src/receiver/ptrace_collector.rs @@ -403,13 +403,9 @@ pub fn capture_thread_context( } /// Maximum time to wait for a single thread to enter ptrace-stop. -/// -/// 200ms accommodates CI environments with heavy CPU contention where the thread -/// may take tens of milliseconds to be scheduled and enter ptrace-stop after -/// PTRACE_INTERRUPT. On unloaded machines the stop typically arrives in <1ms. const STOP_TIMEOUT_PER_THREAD: Duration = Duration::from_millis(200); -/// Delay between retry attempts. Uses an exponential back-off starting from +/// Delay between retry attempts that is used as a base for an exponential back-off starting from /// this value (10ms, 20ms, 40ms). const RETRY_BASE_DELAY: Duration = Duration::from_millis(10); @@ -428,16 +424,14 @@ fn is_transient_ptrace_error(err: &PtraceError) -> bool { /// /// Each attempt gets its own `STOP_TIMEOUT_PER_THREAD` budget (capped at the /// overall deadline) so that a retry after a timeout-induced failure actually -/// has enough time to succeed. On older kernels (e.g. CentOS 7 / kernel 3.10) +/// has enough time to succeed. On older kernels (CentOS 7 / kernel 3.10) /// the first attempt can consume its entire budget waiting for registers to /// become readable; reusing that exhausted deadline would make the retry a /// no-op. /// /// A capture that succeeds but produces zero frames is also retried: on a /// running thread with a confirmed non-zero IP, empty frames indicates a -/// transient libunwind issue (e.g. stale address-space cache state) rather -/// than a permanent problem. Retries use exponential back-off to give the -/// kernel time to fully commit thread state between attempts. +/// transient issue. fn capture_with_retry( tid: libc::pid_t, resolve_frames: crate::StacktraceCollection,