Inject rust stack frames into traceback

bschoenmaeckers · bschoenmaeckers · commit f203b61a309c · 2026-03-13T11:09:17.000+01:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -61,6 +61,7 @@ uuid = { version = "1.12.0", optional = true }
 lock_api = { version = "0.4", optional = true }
 parking_lot = { version = "0.12", optional = true }
 iana-time-zone = { version = "0.1", optional = true, features = ["fallback"]}
+backtrace = "0.3.76"
 
 [target.'cfg(not(target_has_atomic = "64"))'.dependencies]
 portable-atomic = "1.0"
@@ -80,6 +81,7 @@ tempfile = "3.12.0"
 static_assertions = "1.1.0"
 uuid = { version = "1.10.0", features = ["v4"] }
 parking_lot = { version = "0.12.3", features = ["arc_lock"] }
+insta = { version = "1.46.3", features = ["filters"] }
 
 [build-dependencies]
 pyo3-build-config = { path = "pyo3-build-config", version = "=0.28.2", features = ["resolve-config"] }
diff --git a/src/err/err_state.rs b/src/err/err_state.rs
@@ -4,6 +4,8 @@ use std::{
     thread::ThreadId,
 };
 
+#[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+use crate::err::backtrace_to_frames;
 #[cfg(not(Py_3_12))]
 use crate::sync::MutexExt;
 use crate::{
@@ -36,10 +38,15 @@ impl PyErrState {
     }
 
     pub(crate) fn lazy_arguments(ptype: Py<PyAny>, args: impl PyErrArguments + 'static) -> Self {
+        #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+        let backtrace = backtrace::Backtrace::new_unresolved();
+
         Self::from_inner(PyErrStateInner::Lazy(Box::new(move |py| {
             PyErrStateLazyFnOutput {
                 ptype,
                 pvalue: args.arguments(py),
+                #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+                backtrace,
             }
         })))
     }
@@ -301,6 +308,8 @@ impl PyErrStateNormalized {
 pub(crate) struct PyErrStateLazyFnOutput {
     pub(crate) ptype: Py<PyAny>,
     pub(crate) pvalue: Py<PyAny>,
+    #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+    pub(crate) backtrace: backtrace::Backtrace,
 }
 
 pub(crate) type PyErrStateLazyFn =
@@ -390,15 +399,35 @@ fn lazy_into_normalized_ffi_tuple(
 /// This would require either moving some logic from C to Rust, or requesting a new
 /// API in CPython.
 fn raise_lazy(py: Python<'_>, lazy: Box<PyErrStateLazyFn>) {
-    let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py);
+    let PyErrStateLazyFnOutput {
+        ptype,
+        pvalue,
+        #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+        mut backtrace,
+    } = lazy(py);
+
     unsafe {
         if ffi::PyExceptionClass_Check(ptype.as_ptr()) == 0 {
             ffi::PyErr_SetString(
                 PyTypeError::type_object_raw(py).cast(),
                 c"exceptions must derive from BaseException".as_ptr(),
             )
         } else {
-            ffi::PyErr_SetObject(ptype.as_ptr(), pvalue.as_ptr())
+            #[cfg(not(all(debug_assertions, not(Py_LIMITED_API))))]
+            {
+                ffi::PyErr_SetObject(ptype.as_ptr(), pvalue.as_ptr());
+            }
+
+            #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+            {
+                let traceback =
+                    PyTraceback::from_frames(py, None, backtrace_to_frames(py, &mut backtrace))
+                        .ok()
+                        .flatten()
+                        .map_or_else(std::ptr::null_mut, Bound::into_ptr);
+
+                ffi::PyErr_Restore(ptype.into_ptr(), pvalue.into_ptr(), traceback)
+            }
         }
     }
 }
diff --git a/src/err/mod.rs b/src/err/mod.rs
@@ -20,6 +20,8 @@ use crate::{BoundObject, Py, PyAny, Python};
 use err_state::{PyErrState, PyErrStateLazyFnOutput, PyErrStateNormalized};
 use std::convert::Infallible;
 use std::ffi::CStr;
+#[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+use {crate::types::PyFrame, std::ffi::CString};
 
 mod cast_error;
 mod downcast_error;
@@ -127,10 +129,14 @@ impl PyErr {
         T: PyTypeInfo,
         A: PyErrArguments + Send + Sync + 'static,
     {
+        #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+        let backtrace = backtrace::Backtrace::new_unresolved();
         PyErr::from_state(PyErrState::lazy(Box::new(move |py| {
             PyErrStateLazyFnOutput {
                 ptype: T::type_object(py).into(),
                 pvalue: args.arguments(py),
+                #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+                backtrace,
             }
         })))
     }
@@ -289,7 +295,24 @@ impl PyErr {
             Self::print_panic_and_unwind(py, state)
         }
 
-        Some(PyErr::from_state(PyErrState::normalized(state)))
+        let err = PyErr::from_state(PyErrState::normalized(state));
+
+        #[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+        {
+            let mut backtrace = backtrace::Backtrace::new();
+            if let Some(traceback) = PyTraceback::from_frames(
+                py,
+                err.traceback(py),
+                backtrace_to_frames(py, &mut backtrace),
+            )
+            .ok()
+            .flatten()
+            {
+                err.set_traceback(py, Some(traceback));
+            }
+        }
+
+        Some(err)
     }
 
     #[cold]
@@ -696,6 +719,49 @@ impl<'py> IntoPyObject<'py> for PyErr {
     }
 }
 
+#[cfg(all(debug_assertions, not(Py_LIMITED_API)))]
+fn backtrace_to_frames<'py, 'a>(
+    py: Python<'py>,
+    backtrace: &'a mut backtrace::Backtrace,
+) -> impl Iterator<Item = Bound<'py, PyFrame>> + use<'py, 'a> {
+    backtrace.resolve();
+    backtrace
+        .frames()
+        .iter()
+        .flat_map(|frame| frame.symbols())
+        .map(|symbol| (symbol.name().map(|name| format!("{name:#}")), symbol))
+        .skip_while(|(name, _)| {
+            if cfg!(any(target_vendor = "apple", windows)) {
+                // On Apple & Windows platforms, backtrace is not able to remove internal frames
+                // from the backtrace, so we need to skip them manually here.
+                name.as_ref()
+                    .map(|name| name.starts_with("backtrace::"))
+                    .unwrap_or(true)
+            } else {
+                false
+            }
+        })
+        // The first frame is always the capture function, so skip it.
+        .skip(1)
+        .take_while(|(name, _)| {
+            name.as_ref()
+                .map(|name| {
+                    !(name.starts_with("pyo3::impl_::trampoline::")
+                        || name.contains("__rust_begin_short_backtrace"))
+                })
+                .unwrap_or(true)
+        })
+        .filter_map(move |(name, symbol)| {
+            let file =
+                CString::new(symbol.filename()?.as_os_str().to_string_lossy().as_ref()).ok()?;
+
+            let function = CString::new(name.as_deref().unwrap_or("<unknown>")).ok()?;
+            let line = symbol.lineno()?;
+
+            PyFrame::new(py, &file, &function, line as _).ok()
+        })
+}
+
 impl<'py> IntoPyObject<'py> for &PyErr {
     type Target = PyBaseException;
     type Output = Bound<'py, Self::Target>;
@@ -844,6 +910,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(false)]
     fn err_debug() {
         // Debug representation should be like the following (without the newlines):
         // PyErr {
diff --git a/src/types/traceback.rs b/src/types/traceback.rs
@@ -41,6 +41,27 @@ impl PyTraceback {
                 .cast_into_unchecked())
         }
     }
+
+    /// Creates a new traceback object from an iterator of frames.
+    ///
+    /// The frames should be ordered from newest to oldest, i.e. the first frame in the iterator
+    /// will be the innermost frame in the traceback.
+    #[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))]
+    pub fn from_frames<'py, I>(
+        py: Python<'py>,
+        start: Option<Bound<'py, PyTraceback>>,
+        frames: I,
+    ) -> PyResult<Option<Bound<'py, PyTraceback>>>
+    where
+        I: IntoIterator,
+        I::Item: IntoPyObject<'py, Target = PyFrame>,
+    {
+        frames.into_iter().try_fold(start, |prev, frame| {
+            let frame = frame.into_pyobject(py).map_err(Into::into)?.into_bound();
+            let line_number = frame.line_number();
+            PyTraceback::new(py, prev, frame, 0, line_number).map(Some)
+        })
+    }
 }
 
 /// Implementation of functionality for [`PyTraceback`].
@@ -113,6 +134,7 @@ mod tests {
     };
 
     #[test]
+    #[cfg(false)]
     fn format_traceback() {
         Python::attach(|py| {
             let err = py
@@ -196,4 +218,44 @@ def f():
             );
         })
     }
+
+    #[test]
+    #[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))]
+    fn test_insert_traceback() {
+        Python::attach(|py| {
+            // Error happens in rust, so rust frames are created first.
+            let rust_traceback = PyTraceback::from_frames(
+                py,
+                None,
+                [
+                    PyFrame::new(py, c"rust2.rs", c"func2", 22).unwrap(),
+                    PyFrame::new(py, c"rust1.rs", c"func1", 11).unwrap(),
+                ],
+            )
+            .unwrap()
+            .unwrap();
+
+            // Stacktrace where python calls into rust
+            let traceback = PyTraceback::from_frames(
+                py,
+                Some(rust_traceback),
+                [
+                    PyFrame::new(py, c"file2.py", c"func2", 20).unwrap(),
+                    PyFrame::new(py, c"file1.py", c"func1", 10).unwrap(),
+                ],
+            )
+            .unwrap()
+            .unwrap();
+
+            assert_eq!(
+                traceback.format().unwrap(),
+                r#"Traceback (most recent call last):
+  File "file1.py", line 10, in func1
+  File "file2.py", line 20, in func2
+  File "rust1.rs", line 11, in func1
+  File "rust2.rs", line 22, in func2
+"#
+            );
+        })
+    }
 }
diff --git a/tests/snapshots/test_backtrace__rust_frames_in_backtrace@unix.snap b/tests/snapshots/test_backtrace__rust_frames_in_backtrace@unix.snap
@@ -0,0 +1,34 @@
+---
+source: tests/test_backtrace.rs
+expression: traceback
+---
+Traceback (most recent call last):
+  File "[RUST_CORE]/ops/function.rs", line 250, in core::ops::function::FnOnce::call_once
+  File "[RUST_CORE]/ops/function.rs", line 250, in <test_backtrace::test_rust_frames_in_backtrace::{closure#0} as core::ops::function::FnOnce<()>>::call_once
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+  File "./tests/test_backtrace.rs", line 7, in test_backtrace::test_rust_frames_in_backtrace::{closure#0}
+    fn test_rust_frames_in_backtrace() {
+  File "./tests/test_backtrace.rs", line 16, in test_backtrace::test_rust_frames_in_backtrace
+    Python::attach(|py| {
+  File "./src/marker.rs", line 415, in <pyo3::marker::Python>::attach::<test_backtrace::test_rust_frames_in_backtrace::{closure#0}, ()>
+    f(guard.python())
+  File "./tests/test_backtrace.rs", line 24, in test_backtrace::test_rust_frames_in_backtrace::{closure#0}
+    .run(
+  File "./src/marker.rs", line 641, in <pyo3::marker::Python>::run
+    code.run(globals, locals).map(|obj| {
+  File "./src/types/code.rs", line 139, in <pyo3::instance::Bound<pyo3::types::code::PyCode> as pyo3::types::code::PyCodeMethods>::run
+    .assume_owned_or_err(self.py())
+  File "./src/ffi_ptr_ext.rs", line 43, in <*mut pyo3_ffi::object::PyObject as pyo3::ffi_ptr_ext::FfiPtrExt>::assume_owned_or_err
+    unsafe { Bound::from_owned_ptr_or_err(py, self) }
+  File "./src/instance.rs", line 385, in <pyo3::instance::Bound<pyo3::types::any::PyAny>>::from_owned_ptr_or_err
+    None => Err(PyErr::fetch(py)),
+  File "./src/err/mod.rs", line 354, in <pyo3::err::PyErr>::fetch
+    PyErr::take(py).unwrap_or_else(failed_to_fetch)
+  File "<string>", line 4, in <module>
+  File "<string>", line 2, in python_func
+  File "./tests/test_backtrace.rs", line 11, in test_backtrace::test_rust_frames_in_backtrace::__pyfunction_produce_err_result
+    #[pyfunction]
+  File "./tests/test_backtrace.rs", line 13, in test_backtrace::test_rust_frames_in_backtrace::produce_err_result
+    Err(PyValueError::new_err("Error result"))
+  File "./src/exceptions.rs", line 31, in <pyo3::exceptions::PyValueError>::new_err::<&str>
+    $crate::PyErr::new::<$name, A>(args)
diff --git a/tests/snapshots/test_backtrace__rust_frames_in_backtrace@windows.snap b/tests/snapshots/test_backtrace__rust_frames_in_backtrace@windows.snap
@@ -0,0 +1,34 @@
+---
+source: tests/test_backtrace.rs
+expression: traceback
+---
+Traceback (most recent call last):
+  File "[RUST_CORE]\ops\function.rs", line 250, in core::ops::function::FnOnce::call_once
+  File "[RUST_CORE]\ops\function.rs", line 250, in core::ops::function::FnOnce::call_once<test_backtrace::test_rust_frames_in_backtrace::closure_env$0,tuple$<> >
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output;
+  File ".\tests\test_backtrace.rs", line 7, in test_backtrace::test_rust_frames_in_backtrace::closure$0
+    fn test_rust_frames_in_backtrace() {
+  File ".\tests\test_backtrace.rs", line 16, in test_backtrace::test_rust_frames_in_backtrace
+    Python::attach(|py| {
+  File ".\src\marker.rs", line 415, in pyo3::marker::Python::attach<test_backtrace::test_rust_frames_in_backtrace::closure_env$0,tuple$<> >
+    f(guard.python())
+  File ".\tests\test_backtrace.rs", line 24, in test_backtrace::test_rust_frames_in_backtrace::closure$0
+    .run(
+  File ".\src\marker.rs", line 641, in pyo3::marker::Python::run
+    code.run(globals, locals).map(|obj| {
+  File ".\src\types\code.rs", line 139, in pyo3::types::code::impl$1::run
+    .assume_owned_or_err(self.py())
+  File ".\src\ffi_ptr_ext.rs", line 43, in pyo3::ffi_ptr_ext::impl$0::assume_owned_or_err
+    unsafe { Bound::from_owned_ptr_or_err(py, self) }
+  File ".\src\instance.rs", line 385, in pyo3::instance::Bound<pyo3::types::any::PyAny>::from_owned_ptr_or_err
+    None => Err(PyErr::fetch(py)),
+  File ".\src\err\mod.rs", line 354, in pyo3::err::PyErr::fetch
+    PyErr::take(py).unwrap_or_else(failed_to_fetch)
+  File "<string>", line 4, in <module>
+  File "<string>", line 2, in python_func
+  File ".\tests\test_backtrace.rs", line 11, in test_backtrace::test_rust_frames_in_backtrace::__pyfunction_produce_err_result
+    #[pyfunction]
+  File ".\tests\test_backtrace.rs", line 13, in test_backtrace::test_rust_frames_in_backtrace::produce_err_result
+    Err(PyValueError::new_err("Error result"))
+  File ".\src\exceptions.rs", line 31, in pyo3::exceptions::PyValueError::new_err<ref$<str$> >
+    $crate::PyErr::new::<$name, A>(args)
diff --git a/tests/test_backtrace.rs b/tests/test_backtrace.rs
@@ -0,0 +1,46 @@
+#![cfg(all(feature = "macros", not(Py_LIMITED_API)))]
+use insta::assert_snapshot;
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+
+#[test]
+fn test_rust_frames_in_backtrace() {
+    use pyo3::prelude::PyDictMethods;
+    use pyo3::{pyfunction, types::PyDict, Python};
+
+    #[pyfunction]
+    fn produce_err_result() -> PyResult<()> {
+        Err(PyValueError::new_err("Error result"))
+    }
+
+    Python::attach(|py| {
+        let func = wrap_pyfunction!(produce_err_result)(py).unwrap();
+        let globals = PyDict::new(py);
+        globals.set_item("func", func).unwrap();
+
+        let root_dir = format!("{:?}", std::env::current_dir().unwrap());
+
+        let err = py
+            .run(
+                c"def python_func():\n  func()\n\npython_func()",
+                Some(&globals),
+                None,
+            )
+            .unwrap_err();
+
+        let traceback = err.traceback(py).unwrap().format().unwrap();
+
+        insta::with_settings!({
+            snapshot_suffix => std::env::consts::FAMILY,
+            filters => [
+                (root_dir.trim_matches('"'), "."),
+                #[cfg(unix)]
+                ("(?:/[\\w\\-\\.]*)+/library/core/src", "[RUST_CORE]"),
+                #[cfg(windows)]
+                ("(?:(?:/rustc/\\w{40}/)|(?:[\\w\\-.:]*\\\\)+)library\\\\core\\\\src", "[RUST_CORE]"),
+            ],
+        }, {
+            assert_snapshot!(traceback);
+        });
+    });
+}
