DEBUG #4054 clang CI: capture stacks on homing timeout + abort

Three debug-only additions to diagnose the rip-and-test-clang failure
(homing timeout after 60s + Fatal glibc pthread mutex assertion on
shutdown) which my local docker cannot reproduce:

- launch.sh: PYTHONFAULTHANDLER=1, ulimit -c unlimited, LIBC_FATAL_
  STDERR_=1, MALLOC_CHECK_=3 so SIGABRT/SIGSEGV in any Python child
  prints a Python+native stack to stderr (visible via linuxcnc.err).
- qtvcp.py: faulthandler.enable() + register on SIGUSR1 so the smoke
  driver can dump qtvcp's interpreter stack without killing it.
- drive.py: on homing timeout, dump per-joint state, halui machine
  pin, locate qtvcp processes and send SIGUSR1; sleep briefly so the
  stack dump lands in the log before we tear down.

Will be reverted once the clang-only failure mode is understood.

Author: grandixximo

src/emc/usr_intf/qtvcp/qtvcp.py

@@ -8,6 +8,17 @@
 import signal
 import subprocess
 
+# DEBUG #4054 clang CI: register Python faulthandler on SIGUSR1 so the
+# ui-smoke driver can dump the qtvcp interpreter stack on home timeout
+# without killing the process. Default fatal signal set (SIGSEGV/ABRT/
+# BUS/FPE/ILL) is already enabled via PYTHONFAULTHANDLER=1 in launch.sh.
+try:
+    import faulthandler
+    faulthandler.enable()
+    faulthandler.register(signal.SIGUSR1, chain=False)
+except Exception:
+    pass
+
 
 if '--force_pyqt=6' in sys.argv:
     os.environ["QT_API"] = "pyqt6"

tests/ui-smoke/_lib/drive.py

@@ -13,6 +13,7 @@
 
 import argparse
 import linuxcnc
+import os
 import sys
 import time
 
@@ -130,6 +131,41 @@ def home_all(cmd, stat, timeout):
         f"{timeout}s; homed={list(stat.homed[:njoints])} "
         f"task_state={stat.task_state} task_mode={stat.task_mode} "
         f"exec_state={stat.exec_state} njoints={njoints}\n")
+    # DEBUG #4054 clang CI: dump per-joint state + halcmd snapshot +
+    # signal qtvcp processes to dump their Python stacks via faulthandler.
+    try:
+        for i in range(njoints):
+            j = stat.joint[i]
+            sys.stderr.write(
+                f"DEBUG joint[{i}]: homed={j['homed']} homing={j['homing']} "
+                f"enabled={j['enabled']} inpos={j['inpos']} fault={j['fault']} "
+                f"min_hard_limit={j['min_hard_limit']} max_hard_limit={j['max_hard_limit']} "
+                f"min_soft_limit={j['min_soft_limit']} max_soft_limit={j['max_soft_limit']}\n")
+        sys.stderr.write(
+            f"DEBUG axis_mask={stat.axis_mask} kinematics_type={stat.kinematics_type} "
+            f"motion_mode={stat.motion_mode} interp_state={stat.interp_state} "
+            f"estop={stat.estop} enabled={stat.enabled} homed_all={stat.homed}\n")
+    except Exception as e:
+        sys.stderr.write(f"DEBUG joint dump failed: {e}\n")
+    import subprocess
+    for args in (["halcmd", "show", "pin", "halui.machine"],
+                 ["halcmd", "show", "pin", "joint.0"],
+                 ["halcmd", "show", "param", "joint.0"],
+                 ["halcmd", "show", "sig"]):
+        try:
+            out = subprocess.check_output(
+                args, stderr=subprocess.STDOUT, timeout=5).decode()
+            sys.stderr.write(f"DEBUG {' '.join(args)}:\n{out}\n")
+        except Exception as e:
+            sys.stderr.write(f"DEBUG {' '.join(args)} failed: {e}\n")
+    qlog = os.path.expanduser("~/qtdragon.log")
+    if os.path.exists(qlog):
+        try:
+            with open(qlog) as f:
+                tail = f.readlines()[-50:]
+            sys.stderr.write(f"DEBUG qtdragon.log tail:\n{''.join(tail)}\n")
+        except Exception as e:
+            sys.stderr.write(f"DEBUG read {qlog} failed: {e}\n")
     return False
 
 

tests/ui-smoke/_lib/launch.sh

@@ -56,6 +56,40 @@ export GST_PLUGIN_FEATURE_RANK="pulsesink:NONE,alsasink:NONE,osssink:NONE,oss4si
 export PULSE_SERVER=/dev/null
 export SDL_AUDIODRIVER=dummy
 
+# DEBUG #4054 clang CI: enable Python faulthandler so SIGSEGV/SIGABRT/SIGUSR1
+# in a Python process print a native+Python stack to stderr. PYTHONFAULTHANDLER
+# also registers SIGUSR1 via faulthandler.register at interpreter startup when
+# the env var is set, so drive.py can signal qtvcp on homing timeout.
+export PYTHONFAULTHANDLER=1
+# DEBUG #4054 clang CI: enable cores + glibc abort verbosity.
+ulimit -c unlimited
+export LIBC_FATAL_STDERR_=1
+
+# DEBUG #4054 clang CI: wrap qtvcp under gdb --batch so a SIGSEGV inside
+# the Qt event loop (which faulthandler now confirms is happening) gets a
+# C-level backtrace. Only takes effect if gdb is available; gdb is on the
+# GitHub Actions ubuntu-24.04 runner image by default. The wrapper is a
+# temp dir prepended to PATH that shadows qtvcp; the inner gdb invocation
+# uses the absolute path captured BEFORE shadowing so it does not recurse.
+REAL_QTVCP_PATH="$(command -v qtvcp || true)"
+if [ -n "$REAL_QTVCP_PATH" ] && command -v gdb >/dev/null 2>&1; then
+    GDB_WRAP_DIR="$(mktemp -d -t qtvcp-gdb.XXXXXX)"
+    cat >"$GDB_WRAP_DIR/qtvcp" <<WRAP
+#!/bin/bash
+exec gdb -batch -nx \\
+    -ex 'set pagination off' \\
+    -ex 'handle SIG33 nostop noprint pass' \\
+    -ex 'handle SIGCHLD nostop noprint pass' \\
+    -ex 'handle SIGPIPE nostop noprint pass' \\
+    -ex run \\
+    -ex 'echo \n=== signal caught, dumping all-thread backtrace ===\n' \\
+    -ex 'thread apply all bt' \\
+    --args /usr/bin/python3 "$REAL_QTVCP_PATH" "\$@"
+WRAP
+    chmod +x "$GDB_WRAP_DIR/qtvcp"
+    export PATH="$GDB_WRAP_DIR:$PATH"
+fi
+
 # Export the per-invocation values so the inner bash -c receives them
 # as proper env vars (avoids embedding paths into the inner script
 # via quoting, which breaks on apostrophes / spaces).