Collect all threads first pass

Author: gyuheon0h

bin_tests/src/modes/behavior.rs

@@ -138,6 +138,7 @@ pub fn get_behavior(mode_str: &str) -> Box<dyn Behavior> {
         "panic_hook_string" => Box::new(test_014_panic_hook_string::Test),
         "panic_hook_unknown_type" => Box::new(test_015_panic_hook_unknown_type::Test),
         "errno_preservation" => Box::new(test_016_errno_preservation::Test),
+        "multi_thread_collection" => Box::new(test_017_multi_thread_collection::Test),
         "runtime_preload_logger" => Box::new(test_000_donothing::Test),
         _ => panic!("Unknown mode: {mode_str}"),
     }

bin_tests/src/modes/unix/mod.rs

@@ -17,3 +17,4 @@ pub mod test_013_panic_hook_after_fork;
 pub mod test_014_panic_hook_string;
 pub mod test_015_panic_hook_unknown_type;
 pub mod test_016_errno_preservation;
+pub mod test_017_multi_thread_collection;

bin_tests/src/modes/unix/test_017_multi_thread_collection.rs

@@ -0,0 +1,53 @@
+// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+//! Tests that the crashtracker collects stack information for all threads, not
+//! just the crashing thread.
+//!
+//! The test spawns two named background threads that sleep indefinitely, then
+//! crashes the main thread.  The crash report is expected to contain at least
+//! one entry in `error.threads` (the sleeping workers) in addition to the
+//! primary crashing-thread stack in `error.stack`.
+
+use crate::modes::behavior::Behavior;
+use libdd_crashtracker::CrashtrackerConfiguration;
+use std::path::Path;
+use std::thread;
+use std::time::Duration;
+
+pub struct Test;
+
+impl Behavior for Test {
+    fn setup(
+        &self,
+        _output_dir: &Path,
+        config: &mut CrashtrackerConfiguration,
+    ) -> anyhow::Result<()> {
+        // Enable multi-thread collection with a reasonable cap.
+        config.set_collect_all_threads(true);
+        config.set_max_threads(32);
+        Ok(())
+    }
+
+    fn pre(&self, _output_dir: &Path) -> anyhow::Result<()> {
+        Ok(())
+    }
+
+    /// Spawn two named worker threads that sleep until the process crashes.
+    /// The JoinHandles are intentionally leaked so the threads remain alive.
+    fn post(&self, _output_dir: &Path) -> anyhow::Result<()> {
+        for i in 0..2 {
+            let handle = thread::Builder::new()
+                // Linux TASK_COMM_LEN = 16 (including NUL), so max visible chars = 15
+                .name(format!("ct_worker_{i}"))
+                .spawn(move || {
+                    // Sleep long enough to still be alive when the crash fires.
+                    thread::sleep(Duration::from_secs(300));
+                })?;
+            // Leak here. The threads must outlive post() and still be
+            // running when the signal fires
+            std::mem::forget(handle);
+        }
+        Ok(())
+    }
+}

bin_tests/src/test_types.rs

@@ -20,6 +20,7 @@ pub enum TestMode {
     RuntimeCallbackFrameInvalidUtf8,
     RuntimePreloadLogger,
     ErrnoPreservation,
+    MultiThreadCollection,
 }
 
 impl TestMode {
@@ -41,6 +42,7 @@ impl TestMode {
             Self::RuntimeCallbackFrameInvalidUtf8 => "runtime_callback_frame_invalid_utf8",
             Self::RuntimePreloadLogger => "runtime_preload_logger",
             Self::ErrnoPreservation => "errno_preservation",
+            Self::MultiThreadCollection => "multi_thread_collection",
         }
     }
 
@@ -62,6 +64,7 @@ impl TestMode {
             Self::RuntimeCallbackFrameInvalidUtf8,
             Self::RuntimePreloadLogger,
             Self::ErrnoPreservation,
+            Self::MultiThreadCollection,
         ]
     }
 }
@@ -92,6 +95,7 @@ impl std::str::FromStr for TestMode {
             "runtime_callback_frame_invalid_utf8" => Ok(Self::RuntimeCallbackFrameInvalidUtf8),
             "runtime_preload_logger" => Ok(Self::RuntimePreloadLogger),
             "errno_preservation" => Ok(Self::ErrnoPreservation),
+            "multi_thread_collection" => Ok(Self::MultiThreadCollection),
             _ => Err(format!("Unknown test mode: {}", s)),
         }
     }

bin_tests/tests/crashtracker_bin_test.rs

@@ -188,6 +188,97 @@ fn test_crash_tracking_bin_runtime_callback_frame() {
     run_crash_test_with_artifacts(&config, &artifacts_map, &artifacts, validator).unwrap();
 }
 
+/// Tests that when `collect_all_threads` is enabled, the crash report contains
+/// entries in `error.threads` for background threads beyond the crashing thread.
+///
+/// The behavior enables `collect_all_threads`, spawns two named
+/// sleeping worker threads in `post()`, and then crashes the main thread.
+/// We verify:
+///   - `error.threads` is non-empty
+///   - Each thread entry is well-formed: `crashed`, `name`, `stack` present.
+///   - None of the additional threads are marked as crashed (the crashing thread is in
+///     `error.stack`, not `error.threads`).
+///   - The worker thread names are recognizable (ct_worker_0, ct_worker_1).
+#[test]
+#[cfg(target_os = "linux")]
+#[cfg_attr(miri, ignore)]
+fn test_crash_tracking_multi_thread_collection() {
+    let config = CrashTestConfig::new(
+        BuildProfile::Release,
+        TestMode::MultiThreadCollection,
+        CrashType::NullDeref,
+    );
+    let artifacts = StandardArtifacts::new(config.profile);
+    let artifacts_map = fetch_built_artifacts(&artifacts.as_slice()).unwrap();
+
+    let validator: ValidatorFn = Box::new(|payload, _fixtures| {
+        let error = &payload["error"];
+        let threads = error["threads"]
+            .as_array()
+            .expect("error.threads should be a JSON array");
+
+        assert!(
+            !threads.is_empty(),
+            "error.threads should be non-empty when collect_all_threads is enabled; \
+             got payload: {}",
+            serde_json::to_string_pretty(payload).unwrap_or_default()
+        );
+
+        // We spawned 2 workers; there may be additional runtime threads too.
+        // Require at least 1 non-crashing thread captured.
+        let non_crashing: Vec<_> = threads
+            .iter()
+            .filter(|t| !t["crashed"].as_bool().unwrap_or(true))
+            .collect();
+        assert!(
+            !non_crashing.is_empty(),
+            "Expected at least one non-crashing thread in error.threads"
+        );
+
+        // Every thread entry must have the required fields.
+        for thread in threads {
+            assert!(
+                thread["name"].is_string(),
+                "thread entry missing 'name': {thread:?}"
+            );
+            assert!(
+                thread["crashed"].is_boolean(),
+                "thread entry missing 'crashed': {thread:?}"
+            );
+            assert!(
+                thread["stack"].is_object(),
+                "thread entry missing 'stack': {thread:?}"
+            );
+            // crashed must be false for entries in error.threads
+            assert!(
+                !thread["crashed"].as_bool().unwrap_or(true),
+                "threads in error.threads must have crashed=false: {thread:?}"
+            );
+        }
+
+        // At least one of the two named workers should appear.
+        let worker_found = threads.iter().any(|t| {
+            t["name"]
+                .as_str()
+                .map(|n| n.starts_with("ct_worker"))
+                .unwrap_or(false)
+        });
+        assert!(
+            worker_found,
+            "Expected to find at least one ct_worker_N thread in error.threads; \
+             thread names: {:?}",
+            threads
+                .iter()
+                .map(|t| t["name"].as_str().unwrap_or("<none>"))
+                .collect::<Vec<_>>()
+        );
+
+        Ok(())
+    });
+
+    run_crash_test_with_artifacts(&config, &artifacts_map, &artifacts, validator).unwrap();
+}
+
 #[test]
 #[cfg(target_os = "linux")]
 #[cfg_attr(miri, ignore)]

libdd-crashtracker/src/collector/api.rs

@@ -4,10 +4,15 @@
 
 use super::{crash_handler::enable, receiver_manager::Receiver};
 use crate::{
-    clear_spans, clear_traces, collector::crash_handler::register_panic_hook,
-    collector::signal_handler_manager::register_crash_handlers, crash_info::Metadata,
-    reset_counters, shared::configuration::CrashtrackerReceiverConfig, update_config,
-    update_metadata, CrashtrackerConfiguration,
+    clear_spans, clear_traces,
+    collector::crash_handler::register_panic_hook,
+    collector::signal_handler_manager::{
+        register_crash_handlers, register_thread_context_signal_handler,
+    },
+    crash_info::Metadata,
+    reset_counters,
+    shared::configuration::CrashtrackerReceiverConfig,
+    update_config, update_metadata, CrashtrackerConfiguration,
 };
 
 pub static DEFAULT_SYMBOLS: [libc::c_int; 4] =
@@ -60,6 +65,10 @@ pub fn on_fork(
 
     // panic hook is unaffected by fork.
 
+    // Reset the thread context buffer so the forked child starts clean.
+    #[cfg(target_os = "linux")]
+    crate::collector::thread_context_buffer::reset_thread_context_buffer();
+
     update_metadata(metadata)?;
     update_config(config)?;
     Receiver::update_stored_config(receiver_config)?;
@@ -86,6 +95,11 @@ pub fn init(
     Receiver::update_stored_config(receiver_config)?;
     register_crash_handlers(&config)?;
     register_panic_hook()?;
+    if config.collect_all_threads() {
+        #[cfg(target_os = "linux")]
+        crate::collector::thread_context_buffer::init_thread_context_buffer(config.max_threads());
+        register_thread_context_signal_handler()?;
+    }
     enable();
     Ok(())
 }

libdd-crashtracker/src/collector/crash_handler.rs

@@ -301,6 +301,19 @@ fn handle_posix_signal_impl(
 
     let timeout_manager = TimeoutManager::new(config.timeout());
 
+    // Phase 2: collect ucontexts from all other threads before forking, so the
+    // collector child can unwind their stacks.
+    #[cfg(target_os = "linux")]
+    if config.collect_all_threads() {
+        use super::thread_context_buffer::collect_thread_contexts;
+        // Timeout for context collection: half of the overall crash timeout, capped
+        // at 200ms to avoid delaying the crash report excessively.
+        let context_timeout_ms = (config.timeout().as_millis() / 2).min(200) as u64;
+        // SAFETY: current_tid() is the TID of the crashing thread -- we skip it.
+        let crashing_tid = unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t };
+        collect_thread_contexts(crashing_tid, config.max_threads(), context_timeout_ms);
+    }
+
     let receiver = Receiver::from_crashtracker_config(config)?;
 
     let collector = Collector::spawn(