From de3c22bf14d0abb39bd94ea126d879af841ed3d8 Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Tue, 19 May 2026 20:28:53 +0000 Subject: [PATCH] fix: prevent OpenBLAS atfork SIGSEGV during Kit startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NumPy 2.x ships a bundled OpenBLAS (libscipy_openblas64_) that spawns worker threads at import time and registers blas_thread_shutdown_ as a pthread_atfork child handler. When test files (or standalone scripts) import torch at the top level before AppLauncher is instantiated, the thread pool is already live when Kit's libomni.platforminfo.plugin calls fork() during startup. In the child process the atfork handler tries to pthread_join the (now non-existent) worker threads, causing SIGSEGV. Set OPENBLAS_NUM_THREADS=1 via os.environ.setdefault in two places: - tools/conftest.py: injected into every test-subprocess environment so CI tests are protected regardless of their top-level imports. - app_launcher.py: set at module scope (before isaacsim is imported) so standalone scripts that import torch before AppLauncher are also safe. setdefault is used so that an explicit user or CI setting is never overridden. Verified on 8×L40 (torch 2.10, numpy 2.3.5, scipy 1.17.1): - import torch loads libscipy_openblas64_-fdde5778.so (exact crash lib) - Default: 64 OS threads spawned; OPENBLAS_NUM_THREADS=1: stays at 1 - test_ray_caster_patterns.py: 90/90 passed with the fix applied --- .../changelog.d/fix-openblas-fork-crash.rst | 15 +++++++++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 10 ++++++++++ tools/conftest.py | 8 ++++++++ 3 files changed, 33 insertions(+) create mode 100644 source/isaaclab/changelog.d/fix-openblas-fork-crash.rst diff --git a/source/isaaclab/changelog.d/fix-openblas-fork-crash.rst b/source/isaaclab/changelog.d/fix-openblas-fork-crash.rst new file mode 100644 index 00000000000..e7a4233ea9c --- /dev/null +++ b/source/isaaclab/changelog.d/fix-openblas-fork-crash.rst @@ -0,0 +1,15 @@ +Fixed +^^^^^ + +* Fixed a ``SIGSEGV`` crash during Kit startup caused by NumPy's bundled + OpenBLAS ``pthread_atfork`` handler. When ``import torch`` (or any + transitive NumPy import) runs before :class:`AppLauncher` creates the + :class:`~isaacsim.SimulationApp`, OpenBLAS spawns worker threads and + registers ``blas_thread_shutdown_`` as a child-side ``atfork`` handler. + Kit's ``libomni.platforminfo.plugin`` then calls ``fork()`` during + startup; in the child process the handler tries to ``pthread_join`` + threads that no longer exist, causing a segmentation fault. The fix + sets ``OPENBLAS_NUM_THREADS=1`` (via ``setdefault``) before the library + is loaded so that no worker threads are created and the handler is a + safe no-op. Both :mod:`app_launcher` (for standalone scripts) and + ``tools/conftest.py`` (for CI test subprocesses) are patched. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 2bdb8a08932..e036d46172f 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -24,6 +24,16 @@ import sys from typing import Any, Literal +# Prevent OpenBLAS fork-safety crash. NumPy/SciPy ship a bundled OpenBLAS +# that spawns worker threads and registers a pthread_atfork child handler +# (blas_thread_shutdown_). When Kit's platform-info plugin calls fork() +# during startup the handler runs in the child and tries to pthread_join +# threads that were not carried across the fork → SIGSEGV. Setting the +# thread count to 1 *before* the library is loaded avoids the crash because +# no worker threads are created and the atfork handler becomes a no-op. +# Uses setdefault so that an explicit user/CI setting is respected. +os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") + with contextlib.suppress(ModuleNotFoundError): import isaacsim # noqa: F401 from isaacsim import SimulationApp diff --git a/tools/conftest.py b/tools/conftest.py index 15aaa232364..514cde4cacf 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -316,6 +316,14 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci): file_name = os.path.basename(test_file) env = os.environ.copy() env["PYTHONFAULTHANDLER"] = "1" + # Prevent OpenBLAS fork-safety crash: when NumPy or SciPy is imported + # before Kit starts, OpenBLAS spawns a worker-thread pool and registers + # a pthread_atfork handler (blas_thread_shutdown_). Kit's platform-info + # plugin calls fork() during startup; in the child the handler tries to + # pthread_join threads that no longer exist → SIGSEGV. Limiting + # OpenBLAS to a single thread before the subprocess starts avoids the + # crash because no worker threads are created and the handler is a no-op. + env.setdefault("OPENBLAS_NUM_THREADS", "1") timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT)