Support subinterpreters when reporting Python stacks

Iterate over all interpreters in the linked list when determining the PID
offset, fixing an issue where pystack only inspected the first interpreter,
which is not guaranteed to be the main interpreter. This ensures the correct TID
is located even when multiple subinterpreters are present.

Add support for subinterpreters in pure Python stack reporting. Threads running
in subinterpreters are now detected and grouped by interpreter ID. Native stack
reporting for subinterpreters is not yet supported.

Signed-off-by: Saul Cooperman <saulcoops@gmail.com>

Author: Saul Cooperman

Dockerfile

@@ -66,6 +66,9 @@ RUN apt-get update \
     python3.13-dev \
     python3.13-dbg \
     python3.13-venv \
+    python3.14-dev \
+    python3.14-dbg \
+    python3.14-venv \
     make \
     cmake \
     gdb \

news/279.bugfix.rst

@@ -0,0 +1,5 @@
+Fix an issue where the PID offset could not be determined when multiple
+subinterpreters were present. Previously, pystack only checked the first
+interpreter in the linked list, which was not guaranteed to be the main
+interpreter. The fix now iterates over all interpreters and correctly locates
+the TID.

news/279.feature.rst

@@ -0,0 +1,3 @@
+Add support for subinterpreters when reporting pure Python stacks. Threads
+running in subinterpreters are now identified and grouped by interpreter ID.
+Native stack reporting for subinterpreters is not yet supported.

setup.py

@@ -77,6 +77,7 @@
         "src/pystack/_pystack.pyx",
         "src/pystack/_pystack/corefile.cpp",
         "src/pystack/_pystack/elf_common.cpp",
+        "src/pystack/_pystack/interpreter.cpp",
         "src/pystack/_pystack/logging.cpp",
         "src/pystack/_pystack/mem.cpp",
         "src/pystack/_pystack/process.cpp",

src/pystack/__init__.py

@@ -1,7 +1,5 @@
 from ._version import __version__
-from .traceback_formatter import print_thread
 
 __all__ = [
     "__version__",
-    "print_thread",
 ]

src/pystack/__main__.py

@@ -19,13 +19,13 @@
 from pystack.process import is_gzip
 
 from . import errors
-from . import print_thread
 from .colors import colored
 from .engine import CoreFileAnalyzer
 from .engine import NativeReportingMode
 from .engine import StackMethod
 from .engine import get_process_threads
 from .engine import get_process_threads_for_core
+from .traceback_formatter import TracebackPrinter
 
 PERMISSION_ERROR_MSG = "Operation not permitted"
 NO_SUCH_PROCESS_ERROR_MSG = "No such process"
@@ -285,14 +285,17 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) ->
     if not args.block and args.native_mode != NativeReportingMode.OFF:
         parser.error("Native traces are only available in blocking mode")
 
+    printer = TracebackPrinter(
+        native_mode=args.native_mode, include_subinterpreters=True
+    )
     for thread in get_process_threads(
         args.pid,
         stop_process=args.block,
         native_mode=args.native_mode,
         locals=args.locals,
         method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO,
     ):
-        print_thread(thread, args.native_mode)
+        printer.print_thread(thread)
 
 
 def format_psinfo_information(psinfo: Dict[str, Any]) -> str:
@@ -414,6 +417,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N
                 elf_id if elf_id else "<MISSING>",
             )
 
+    printer = TracebackPrinter(args.native_mode, include_subinterpreters=True)
     for thread in get_process_threads_for_core(
         corefile,
         executable,
@@ -422,7 +426,7 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N
         locals=args.locals,
         method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO,
     ):
-        print_thread(thread, args.native_mode)
+        printer.print_thread(thread)
 
 
 if __name__ == "__main__":  # pragma: no cover

src/pystack/_pystack.pyx

@@ -22,6 +22,7 @@ from _pystack.elf_common cimport CoreFileAnalyzer as NativeCoreFileAnalyzer
 from _pystack.elf_common cimport ProcessAnalyzer as NativeProcessAnalyzer
 from _pystack.elf_common cimport SectionInfo
 from _pystack.elf_common cimport getSectionInfo
+from _pystack.interpreter cimport InterpreterUtils
 from _pystack.logging cimport initializePythonLoggerInterface
 from _pystack.mem cimport AbstractRemoteMemoryManager
 from _pystack.mem cimport MemoryMapInformation as CppMemoryMapInformation
@@ -462,6 +463,7 @@ cdef object _construct_threads_from_interpreter_state(
     bint add_native_traces,
     bint resolve_locals,
 ):
+    interpreter_id = InterpreterUtils.getInterpreterId(manager, head)
     LOGGER.info("Fetching Python threads")
     threads = []
 
@@ -486,6 +488,7 @@ cdef object _construct_threads_from_interpreter_state(
                 current_thread.isGilHolder(),
                 current_thread.isGCCollecting(),
                 python_version,
+                interpreter_id,
                 name=get_thread_name(pid, current_thread.Tid()),
             )
         )
@@ -622,7 +625,7 @@ def _get_process_threads(
         )
 
     all_tids = list(manager.get().Tids())
-    if head:
+    while head:
         add_native_traces = native_mode != NativeReportingMode.OFF
         for thread in _construct_threads_from_interpreter_state(
             manager,
@@ -635,6 +638,7 @@ def _get_process_threads(
             if thread.tid in all_tids:
                 all_tids.remove(thread.tid)
             yield thread
+        head = InterpreterUtils.getNextInterpreter(manager, head)
 
     if native_mode == NativeReportingMode.ALL:
         yield from _construct_os_threads(manager, pid, all_tids)
@@ -769,14 +773,20 @@ def _get_process_threads_for_core(
 
     all_tids = list(manager.get().Tids())
 
-    if head:
-        native = native_mode in {NativeReportingMode.PYTHON, NativeReportingMode.ALL}
+    while head:
+        add_native_traces = native_mode != NativeReportingMode.OFF
         for thread in _construct_threads_from_interpreter_state(
-            manager, head, pymanager.pid, pymanager.python_version, native, locals
+            manager,
+            head,
+            pymanager.pid,
+            pymanager.python_version,
+            add_native_traces,
+            locals,
         ):
             if thread.tid in all_tids:
                 all_tids.remove(thread.tid)
             yield thread
+        head = InterpreterUtils.getNextInterpreter(manager, head)
 
     if native_mode == NativeReportingMode.ALL:
         yield from _construct_os_threads(manager, pymanager.pid, all_tids)

src/pystack/_pystack/CMakeLists.txt

@@ -21,6 +21,7 @@ add_library(_pystack STATIC
             pythread.cpp
             version.cpp
             elf_common.cpp
-            pytypes.cpp)
+            pytypes.cpp
+            interpreter.cpp)
 set_property(TARGET _pystack PROPERTY POSITION_INDEPENDENT_CODE ON)
 include_directories("." "cpython" ${PYTHON_INCLUDE_DIRS})

src/pystack/_pystack/cpython/interpreter.h

@@ -375,10 +375,10 @@ struct _gil_runtime_state
     int locked;
     unsigned long switch_number;
     pthread_cond_t cond;
-    pthread_cond_t mutex;
+    pthread_mutex_t mutex;
 #ifdef FORCE_SWITCHING
     pthread_cond_t switch_cond;
-    pthread_cond_t switch_mutex;
+    pthread_mutex_t switch_mutex;
 #endif
 };
 

src/pystack/_pystack/interpreter.cpp

@@ -0,0 +1,36 @@
+#include <memory>
+
+#include "interpreter.h"
+#include "logging.h"
+#include "process.h"
+#include "structure.h"
+#include "version.h"
+
+namespace pystack {
+
+remote_addr_t
+InterpreterUtils::getNextInterpreter(
+        const std::shared_ptr<const AbstractProcessManager>& manager,
+        remote_addr_t interpreter_addr)
+{
+    Structure<py_is_v> is(manager, interpreter_addr);
+    return is.getField(&py_is_v::o_next);
+}
+
+int64_t
+InterpreterUtils::getInterpreterId(
+        const std::shared_ptr<const AbstractProcessManager>& manager,
+        remote_addr_t interpreter_addr)
+{
+    if (!manager->versionIsAtLeast(3, 7)) {
+        // No support for subinterpreters so the only interpreter is ID 0.
+        return 0;
+    }
+
+    Structure<py_is_v> is(manager, interpreter_addr);
+    int64_t id_value = is.getField(&py_is_v::o_id);
+
+    return id_value;
+}
+
+}  // namespace pystack