feat(crashtracking)!: collect all threads

bin_tests/src/modes/behavior.rs

@@ -138,6 +138,7 @@ pub fn get_behavior(mode_str: &str) -> Box<dyn Behavior> {
         "panic_hook_string" => Box::new(test_014_panic_hook_string::Test),
         "panic_hook_unknown_type" => Box::new(test_015_panic_hook_unknown_type::Test),
         "errno_preservation" => Box::new(test_016_errno_preservation::Test),
+        "multi_thread_collection" => Box::new(test_017_multi_thread_collection::Test),
         "runtime_preload_logger" => Box::new(test_000_donothing::Test),
         _ => panic!("Unknown mode: {mode_str}"),
     }

bin_tests/src/modes/unix/mod.rs

@@ -17,3 +17,4 @@ pub mod test_013_panic_hook_after_fork;
 pub mod test_014_panic_hook_string;
 pub mod test_015_panic_hook_unknown_type;
 pub mod test_016_errno_preservation;
+pub mod test_017_multi_thread_collection;

bin_tests/src/modes/unix/test_017_multi_thread_collection.rs

@@ -0,0 +1,94 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+//! Tests that the crashtracker collects stack information for all threads, not
+//! just the crashing thread.
+//!
+//! Two named background threads are spawned with distinct, recognisable call
+//! chains so that the captured stacks are visually interesting and clearly
+//! distinguishable in the crash report.
+//!
+//! ct_worker_0: worker_entry_0 -> wait_for_work_0 -> thread::sleep
+//! ct_worker_1: worker_entry_1 -> wait_for_work_1 -> thread::sleep
+//!
+//! All intermediate functions are #[inline(never)] so they appear as distinct
+//! frames in the libunwind output.
+
+use crate::modes::behavior::Behavior;
+use libdd_crashtracker::CrashtrackerConfiguration;
+use std::path::Path;
+use std::sync::{Arc, Barrier};
+use std::thread;
+use std::time::Duration;
+
+pub struct Test;
+
+#[inline(never)]
+fn wait_for_work_0() {
+    let _ = std::hint::black_box(1u64); // prevent the compiler from optimizing out the sleep
+    thread::sleep(Duration::from_secs(300));
+}
+
+#[inline(never)]
+fn worker_entry_0() {
+    let _ = std::hint::black_box(1u64);
+    wait_for_work_0();
+}
+
+#[inline(never)]
+fn wait_for_work_1() {
+    let _ = std::hint::black_box(1u64); // prevent the compiler from optimizing out the sleep
+    thread::sleep(Duration::from_secs(300));
+}
+
+#[inline(never)]
+fn worker_entry_1() {
+    let _ = std::hint::black_box(1u64);
+    wait_for_work_1();
+}
+
+impl Behavior for Test {
+    fn setup(
+        &self,
+        _output_dir: &Path,
+        config: &mut CrashtrackerConfiguration,
+    ) -> anyhow::Result<()> {
+        config.set_collect_all_threads(true);
+        config.set_max_threads(32);
+        Ok(())
+    }
+
+    fn pre(&self, _output_dir: &Path) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    /// Spawn two named worker threads with distinct call chains, then leak the
+    /// handles so the threads outlive post() and are still live when the crash fires.
+    fn post(&self, _output_dir: &Path) -> anyhow::Result<()> {
+        // 2 workers + 1 for this (main) thread.
+        let barrier = Arc::new(Barrier::new(3));
+
+        let b0 = Arc::clone(&barrier);
+        let h0 = thread::Builder::new()
+            .name("ct_worker_0".to_string())
+            .spawn(move || {
+                b0.wait();
+                worker_entry_0();
+            })?;
+
+        let b1 = Arc::clone(&barrier);
+        let h1 = thread::Builder::new()
+            .name("ct_worker_1".to_string())
+            .spawn(move || {
+                b1.wait();
+                worker_entry_1();
+            })?;
+
+        barrier.wait();
+        thread::sleep(Duration::from_millis(20));
+
+        std::mem::forget(h0);
+        std::mem::forget(h1);
+        Ok(())
+    }
+}

bin_tests/src/test_types.rs

@@ -20,6 +20,7 @@ pub enum TestMode {
     RuntimeCallbackFrameInvalidUtf8,
     RuntimePreloadLogger,
     ErrnoPreservation,
+    MultiThreadCollection,
 }
 
 impl TestMode {
@@ -41,6 +42,7 @@ impl TestMode {
             Self::RuntimeCallbackFrameInvalidUtf8 => "runtime_callback_frame_invalid_utf8",
             Self::RuntimePreloadLogger => "runtime_preload_logger",
             Self::ErrnoPreservation => "errno_preservation",
+            Self::MultiThreadCollection => "multi_thread_collection",
         }
     }
 
@@ -62,6 +64,7 @@ impl TestMode {
             Self::RuntimeCallbackFrameInvalidUtf8,
             Self::RuntimePreloadLogger,
             Self::ErrnoPreservation,
+            Self::MultiThreadCollection,
         ]
     }
 }
@@ -92,6 +95,7 @@ impl std::str::FromStr for TestMode {
             "runtime_callback_frame_invalid_utf8" => Ok(Self::RuntimeCallbackFrameInvalidUtf8),
             "runtime_preload_logger" => Ok(Self::RuntimePreloadLogger),
             "errno_preservation" => Ok(Self::ErrnoPreservation),
+            "multi_thread_collection" => Ok(Self::MultiThreadCollection),
             _ => Err(format!("Unknown test mode: {}", s)),
         }
     }

bin_tests/tests/crashtracker_bin_test.rs

@@ -1,4 +1,4 @@
 // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
 #![cfg(unix)]
@@ -188,6 +188,135 @@
     run_crash_test_with_artifacts(&config, &artifacts_map, &artifacts, validator).unwrap();
 }
 
+/// Tests that when `collect_all_threads` is enabled, the crash report contains
+/// entries in `error.threads` for background threads beyond the crashing thread.
+///
+/// The behavior (test_017_multi_thread_collection.rs) enables `collect_all_threads`,
+/// spawns two named sleeping worker threads in `post()`, and then crashes the main thread.
+///
+/// Thread collection now happens in the receiver process using libunwind remote unwinding
+/// via ptrace (_UPT_create / unw_init_remote / unw_step_remote). The parent process stays
+/// alive until the receiver completes, guaranteeing threads are valid ptrace targets.
+///
+/// We verify:
+///   - `error.threads` is non-empty.
+///   - Each thread entry is well-formed: `crashed=false`, `name`, and `stack` present.
+///   - None of the additional threads are marked as crashed (the crashing thread is in
+///     `error.stack`, not `error.threads`).
+///   - Both worker threads are present by name (ct_worker_0, ct_worker_1).
+///   - Each worker has a multi-frame stack trace including their entry function, confirming that
+///     libunwind remote unwinding produced a full call chain rather than a single syscall frame.
+#[test]
+#[cfg(target_os = "linux")]
+#[cfg_attr(miri, ignore)]
+fn test_crash_tracking_multi_thread_collection() {
+    let config = CrashTestConfig::new(
+        BuildProfile::Release,
+        TestMode::MultiThreadCollection,
+        CrashType::NullDeref,
+    );
+    let artifacts = StandardArtifacts::new(config.profile);
+    let artifacts_map = fetch_built_artifacts(&artifacts.as_slice()).unwrap();
+
+    let validator: ValidatorFn = Box::new(|payload, _fixtures| {
+        let error = &payload["error"];
+        // assert!(
+        //     false,
+        //     "{}",
+        //     serde_json::to_string_pretty(error).unwrap_or_default()
+        // );
+        let threads = error["threads"]
+            .as_array()
+            .expect("error.threads should be a JSON array");
+
+        let thread_names: Vec<&str> = threads
+            .iter()
+            .map(|t| t["name"].as_str().unwrap_or("<none>"))
+            .collect();
+
+        assert!(
+            !threads.is_empty(),
+            "error.threads should be non-empty when collect_all_threads is enabled; \
+             got payload: {}",
+            serde_json::to_string_pretty(payload).unwrap_or_default()
+        );
+
+        // Every thread entry must be structurally valid and non-crashing
+        for thread in threads {
+            assert!(
+                thread["name"].is_string(),
+                "thread entry missing 'name': {thread:?}"
+            );
+            assert!(
+                thread["crashed"].is_boolean(),
+                "thread entry missing 'crashed': {thread:?}"
+            );
+            assert!(
+                thread["stack"].is_object(),
+                "thread entry missing 'stack': {thread:?}"
+            );
+            assert!(
+                !thread["crashed"].as_bool().unwrap_or(true),
+                "threads in error.threads must have crashed=false: {thread:?}"
+            );
+        }
+
+        // Both named workers must be present; the behavior spawns exactly two
+        for expected in ["ct_worker_0", "ct_worker_1"] {
+            assert!(
+                thread_names.contains(&expected),
+                "Expected worker thread '{expected}' in error.threads; \
+                 got: {thread_names:?}"
+            );
+        }
+
+        // Each worker must have a multi-frame stack trace.
+        //
+        // The workers sleep in thread::sleep -> wait_for_work_N -> worker_entry_N.
+        // With libunwind remote unwinding, we expect the full call chain rather than
+        // a single syscall frame. We verify:
+        //   - More than one frame was captured.
+        //   - At least one frame contains the worker's entry function by name.
+        for expected in ["ct_worker_0", "ct_worker_1"] {
+            let worker = threads
+                .iter()
+                .find(|t| t["name"].as_str() == Some(expected))
+                .unwrap_or_else(|| panic!("{expected} should be in threads"));
+
+            let frames = worker["stack"]["frames"]
+                .as_array()
+                .unwrap_or_else(|| panic!("{expected} stack.frames should be an array"));
+
+            assert!(
+                frames.len() > 1,
+                "{expected} should have a multi-frame stack trace from remote libunwind; \
+                 got {} frame(s): {frames:?}",
+                frames.len()
+            );
+
+            let entry_fn = if expected == "ct_worker_0" {
+                "worker_entry_0"
+            } else {
+                "worker_entry_1"
+            };
+            let has_entry_frame = frames.iter().any(|f| {
+                f["function"]
+                    .as_str()
+                    .map(|name| name.contains(entry_fn))
+                    .unwrap_or(false)
+            });
+            assert!(
+                has_entry_frame,
+                "{expected} stack should contain a frame for '{entry_fn}' but got: {frames:?}"
+            );
+        }
+
+        Ok(())
+    });
+
+    run_crash_test_with_artifacts(&config, &artifacts_map, &artifacts, validator).unwrap();
+}
+
 #[test]
 #[cfg(target_os = "linux")]
 #[cfg_attr(miri, ignore)]

libdd-crashtracker/Cargo.toml

@@ -42,7 +42,7 @@ cxx = ["dep:cxx", "dep:cxx-build"]
 blazesym = "=0.2.3"
 
 [target.'cfg(target_os = "linux")'.dependencies]
-libdd-libunwind-sys = { version = "1.0.0" }
+libdd-libunwind-sys = { git = "https://github.com/DataDog/libdatadog-libunwind", branch = "gyuheon0h/remote-unwind-api" }
 
 [dependencies]
 anyhow = "1.0"

libdd-crashtracker/src/collector/collector_manager.rs

@@ -57,8 +57,8 @@ impl Collector {
                     tid,
                 );
             }
-            pid if pid > 0 => Ok(Self {
-                handle: ProcessHandle::new(receiver.handle.uds_fd, Some(pid)),
+            child_pid if child_pid > 0 => Ok(Self {
+                handle: ProcessHandle::new(receiver.handle.uds_fd, Some(child_pid)),
             }),
             code => {
                 // Error

libdd-crashtracker/src/collector/crash_handler.rs

@@ -303,6 +303,18 @@ fn handle_posix_signal_impl(
 
     let receiver = Receiver::from_crashtracker_config(config)?;
 
+    // Enable ptrace permissions for receiver if multi-thread collection is enabled
+    #[cfg(target_os = "linux")]
+    if config.collect_all_threads() {
+        if let Some(receiver_pid) = receiver.handle.pid {
+            // Allow the receiver to ptrace this process for thread context collection
+            // SAFETY: prctl is async-signal-safe and we're just setting ptrace permissions
+            unsafe {
+                libc::prctl(libc::PR_SET_PTRACER, receiver_pid as libc::c_ulong, 0, 0, 0);
+            }
+        }
+    }
+
     let collector = Collector::spawn(
         &receiver,
         config,

libdd-crashtracker/src/receiver/mod.rs

@@ -7,6 +7,8 @@ pub use entry_points::{
     async_receiver_entry_point_unix_listener, async_receiver_entry_point_unix_socket,
     get_receiver_unix_socket, receiver_entry_point_stdin, receiver_entry_point_unix_socket,
 };
+#[cfg(target_os = "linux")]
+mod ptrace_collector;
 mod receive_report;
 
 #[cfg(feature = "benchmarking")]