Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions ddtrace/internal/native/_native.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,15 @@ class SharedRuntime:
def shutdown(self, timeout_ms: Optional[int] = None) -> None:
"""Gracefully shut down the shared runtime.

Args:
timeout_ms: Maximum time in milliseconds to wait for shutdown.
If None, waits indefinitely.
"""
...
def shutdown_in_thread(self, timeout_ms: Optional[int] = None) -> None:
"""Gracefully shut down the shared runtime.
The code is run in a separate thread to bypass a stale thread local storage.

Args:
timeout_ms: Maximum time in milliseconds to wait for shutdown.
If None, waits indefinitely.
Expand Down
8 changes: 7 additions & 1 deletion ddtrace/internal/native_runtime.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import importlib.util
import logging
from typing import Optional

Expand Down Expand Up @@ -28,6 +29,7 @@ def __init__(self) -> None:
forksafe.register_after_parent(self.after_fork_parent)
forksafe.register(self.after_fork_child)
atexit.register(self._atexit)
atexit.register_on_exit_signal(self._atexit)

def _atexit(self) -> None:
try:
Expand All @@ -43,7 +45,11 @@ def shutdown(self, timeout_ms: Optional[int] = None) -> None:
If None, waits indefinitely — only safe if all workers have
already been stopped (e.g. via TraceExporter.shutdown).
"""
super().shutdown(timeout_ms=timeout_ms)
using_uwsgi = importlib.util.find_spec("uwsgi") is not None

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Use import-based uWSGI detection

When running under uWSGI on Python versions where the server injects the synthetic uwsgi module without a module spec, importlib.util.find_spec("uwsgi") raises ValueError instead of returning a truthy value. Since _atexit() catches that exception, SIGTERM exits skip shutdown_in_thread() entirely, so the uWSGI crash path this change is meant to fix still uses no native-runtime shutdown; use import uwsgi/sys.modules or the existing uWSGI helper and handle ValueError here.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍🏻 I agree with this feedback, I thik uwsgi directly injects the module in sys.modules, so doing a pyimport of uwsgi or checking sys.modules["uwsgi"] would be better

if not using_uwsgi:
super().shutdown(timeout_ms=timeout_ms)
else:
super().shutdown_in_thread(timeout_ms=timeout_ms)
atexit.unregister(self._atexit)
forksafe.unregister_before_fork(self.before_fork)
forksafe.unregister_parent(self.after_fork_parent)
Expand Down
4 changes: 4 additions & 0 deletions ddtrace/internal/writer/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,9 @@ def set_test_session_token(self, token: Optional[str]) -> None:
:param token: The test session token to use for authentication.
"""
self._test_session_token = token
old_exporter = self._exporter
self._exporter = self._create_exporter()
old_exporter.shutdown(3_000_000_000)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Stop exporters when recreating stopped writers

This new explicit shutdown covers token/API swaps, but NativeWriter.recreate() still catches ServiceStatusError and returns a new writer without stopping the old exporter when the writer was never started, which happens in fork-child recreation and LLMObs/AppSec reconfiguration before the first trace. Since this commit removed the Rust Drop shutdown fallback, those old exporters leave their SharedRuntime workers registered/running until process exit, causing duplicate native workers after each such recreate; add the same exporter shutdown/drop cleanup on the stopped-writer recreate path.

Useful? React with 👍 / 👎.


def recreate(
self,
Expand Down Expand Up @@ -890,7 +892,9 @@ def _downgrade(self, status, client):
if client.ENDPOINT == "v0.5/traces":
self._clients = [AgentWriterClientV4(self._buffer_size, self._max_payload_size)]
self._api_version = "v0.4"
old_exporter = self._exporter
self._exporter = self._create_exporter()
old_exporter.shutdown(3_000_000_000)

# Since we have to change the encoding in this case, the payload
# would need to be converted to the downgraded encoding before
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
fixes:
- |
Fix crashes in uwsgi worker when exiting from SIGTERM.
8 changes: 0 additions & 8 deletions src/native/data_pipeline/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,14 +269,6 @@ impl TraceExporterPy {
}
}

impl Drop for TraceExporterPy {
fn drop(&mut self) {
if let Some(exporter) = self.inner.take() {
let _ = exporter.shutdown(Some(Duration::from_secs(3)));
}
}
}

#[pymodule]
pub fn register_data_pipeline(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<TraceExporterBuilderPy>()?;
Expand Down
14 changes: 14 additions & 0 deletions src/native/shared_runtime/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use libdd_shared_runtime::SharedRuntime;
use pyo3::prelude::*;
use std::sync::Arc;
use std::thread;
use std::time::Duration;

mod exceptions;
Expand Down Expand Up @@ -51,6 +52,19 @@ impl SharedRuntimePy {
.map_err(shared_runtime_error_to_pyerr)
}

/// Shutdown the runtime in a new thread.
/// This is can be used when thread local storage have been destroyed.
fn shutdown_in_thread(&self, timeout_ms: Option<u64>) -> PyResult<()> {
let timeout = timeout_ms.map(Duration::from_millis);
let inner = self.inner.clone();
let result = thread::spawn(move || inner.shutdown(timeout))
.join()
.map_err(|_| {
pyo3::exceptions::PyRuntimeError::new_err("Failed to join shutdown thread")
})?;
result.map_err(shared_runtime_error_to_pyerr)
}

fn debug(&self) -> String {
format!("{:?}", self.inner)
}
Expand Down
Loading