*shared* druntime: Avoid aborting with '_handleToDSO not in sync with _loadedDSOs' (#5048)

This error currently happens when a D DSO is unloaded in a thread the DSO
wasn't registered with. So for example, loading a D Python extension DLL
via dlopen/LoadLibrary in a Python worker thread (dragging in the druntime
and Phobos DLLs etc., and initializing those in the worker thread), and
one of the D DLLs then being unloaded in the Python *main* thread at
Python exit.

druntime has no way of registering the unknown main thread with druntime
(registering the OS-initialized TLS data with the D GC, running the TLS
module ctors etc.). [Normally, if you spawn a new thread via druntime,
then that new thread inherits all the DSOs registered with the parent
thread.]

What we currently do in case a DSO is unloaded in such a thread is still
running the TLS module dtors (as well as the shared module dtors etc.) of
that DSO, but then printing that 'not in sync' message and aborting the
process, so not finalizing any remaining DSOs.

Try to handle such scenarios more gracefully by
* emitting a stderr warning (per unloaded DSO) instead of aborting (and
  so not skipping any remaining DSOs anymore, as well as not tampering
  with the exit code),
* emitting the warning *before* the DSO finalization, so that any
  issues there are easier to troubleshoot, and
* excluding the TLS module dtors for such not-registered-in-current-thread
  DSOs, as the TLS module ctors most likely haven't run either in that
  thread.

Author: kinke

runtime/druntime/src/rt/sections_elf_shared.d

@@ -645,54 +645,73 @@ package extern(C) void _d_dso_registry(void* arg)
             if (doFinalize)
                 doFinalize = !gc_isProxied();
         }
-        if (doFinalize)
-        {
-            // rt_unloadLibrary already ran tls dtors, so do this only for dlclose
-            immutable runTlsDtors = !_rtLoading;
-            runModuleDestructors(pdso, runTlsDtors);
-            unregisterGCRanges(pdso);
-            // run finalizers after module dtors (same order as in rt_term)
-            version (Shared) runFinalizers(pdso);
-        }
 
+        bool runTlsDtors = true;
         version (Shared)
         {
-            if (!_rtLoading)
+            size_t loadedDSOIndex = size_t.max;
+
+            if (_rtLoading)
+            {
+                // This DSO is being unloaded via rt_unloadLibrary; the TLS
+                // module dtors already ran, and it was removed from _loadedDSOs.
+                runTlsDtors = false;
+            }
+            else // dlclose
             {
-                /* This DSO was not unloaded by rt_unloadLibrary so we
-                 * have to remove it from _loadedDSOs here.
-                 */
                 foreach (i, ref tdso; _loadedDSOs)
                 {
                     if (tdso._pdso == pdso)
                     {
-                        _loadedDSOs.remove(i);
+                        loadedDSOIndex = i;
                         break;
                     }
                 }
+
+                if (loadedDSOIndex == size_t.max)
+                {
+                    import core.stdc.stdio : fprintf, stderr;
+                    fprintf(stderr, "druntime warning: DSO being unloaded isn't in thread-local DSO list. Terminating/unloading in a thread not attached to druntime?\n");
+                    runTlsDtors = false;
+                }
             }
+        }
 
-            unsetDSOForHandle(pdso, pdso._handle);
+        if (doFinalize)
+        {
+            runModuleDestructors(pdso, runTlsDtors);
+            unregisterGCRanges(pdso);
+            // run finalizers after module dtors (same order as in rt_term)
+            version (Shared) runFinalizers(pdso);
+        }
+
+        version (Shared)
+        {
+            if (loadedDSOIndex != size_t.max)
+            {
+                safeAssert(_loadedDSOs[loadedDSOIndex]._pdso == pdso, "Stale loadedDSOIndex.");
+                _loadedDSOs.remove(loadedDSOIndex);
+            }
+
+            const numGlobalDSOs = unsetDSOForHandle(pdso, pdso._handle);
+            safeAssert(_loadedDSOs.length <= numGlobalDSOs, "Thread-local DSO list with more entries than thread-global one.");
+            const isLastDSO = numGlobalDSOs == 0;
+            if (isLastDSO)
+                _handleToDSO.reset();
         }
         else
         {
             // static DSOs are unloaded in reverse order
             safeAssert(pdso == _loadedDSOs.back, "DSO being unregistered isn't current last one.");
             _loadedDSOs.popBack();
+            const isLastDSO = _loadedDSOs.empty;
         }
 
         freeDSO(pdso);
 
         // last DSO being unloaded => shutdown registry
-        if (_loadedDSOs.empty)
-        {
-            version (Shared)
-            {
-                safeAssert(_handleToDSO.empty, "_handleToDSO not in sync with _loadedDSOs.");
-                _handleToDSO.reset();
-            }
+        if (isLastDSO)
             finiLocks();
-        }
     }
 }
 
@@ -898,12 +917,14 @@ version (Shared)
         _handleToDSOMutex.unlock_nothrow();
     }
 
-    void unsetDSOForHandle(DSO* pdso, void* handle)
+    size_t unsetDSOForHandle(DSO* pdso, void* handle)
     {
         _handleToDSOMutex.lock_nothrow();
         safeAssert(_handleToDSO[handle] == pdso, "Handle doesn't match registered DSO.");
         _handleToDSO.remove(handle);
+        const numDSOs = _handleToDSO.length;
         _handleToDSOMutex.unlock_nothrow();
+        return numDSOs;
     }
 
     static if (SharedELF) void getDependencies(const scope ref SharedObject object, ref Array!(DSO*) deps)