DEBUG #4054 clang CI: capture stacks on homing timeout + abort

Three debug-only additions to diagnose the rip-and-test-clang failure
(homing timeout after 60s + Fatal glibc pthread mutex assertion on
shutdown) which my local docker cannot reproduce:

- launch.sh: PYTHONFAULTHANDLER=1, ulimit -c unlimited, LIBC_FATAL_
  STDERR_=1, MALLOC_CHECK_=3 so SIGABRT/SIGSEGV in any Python child
  prints a Python+native stack to stderr (visible via linuxcnc.err).
- qtvcp.py: faulthandler.enable() + register on SIGUSR1 so the smoke
  driver can dump qtvcp's interpreter stack without killing it.
- drive.py: on homing timeout, dump per-joint state, halui machine
  pin, locate qtvcp processes and send SIGUSR1; sleep briefly so the
  stack dump lands in the log before we tear down.

Will be reverted once the clang-only failure mode is understood.

Author: grandixximo

src/emc/usr_intf/qtvcp/qtvcp.py

@@ -8,6 +8,15 @@
 import signal
 import subprocess
 
+# DEBUG #4054 clang CI bisect: faulthandler.enable() was a candidate
+# masker of the qtvcp:481 Abort/Segfault. Temporarily disabled to test.
+# try:
+#     import faulthandler
+#     faulthandler.enable()
+#     faulthandler.register(signal.SIGUSR1, chain=False)
+# except Exception:
+#     pass
+
 
 if '--force_pyqt=6' in sys.argv:
     os.environ["QT_API"] = "pyqt6"

tests/ui-smoke/_lib/drive.py

@@ -13,6 +13,7 @@
 
 import argparse
 import linuxcnc
+import os
 import sys
 import time
 
@@ -130,6 +131,41 @@ def home_all(cmd, stat, timeout):
         f"{timeout}s; homed={list(stat.homed[:njoints])} "
         f"task_state={stat.task_state} task_mode={stat.task_mode} "
         f"exec_state={stat.exec_state} njoints={njoints}\n")
+    # DEBUG #4054 clang CI: dump per-joint state + halcmd snapshot +
+    # signal qtvcp processes to dump their Python stacks via faulthandler.
+    try:
+        for i in range(njoints):
+            j = stat.joint[i]
+            sys.stderr.write(
+                f"DEBUG joint[{i}]: homed={j['homed']} homing={j['homing']} "
+                f"enabled={j['enabled']} inpos={j['inpos']} fault={j['fault']} "
+                f"min_hard_limit={j['min_hard_limit']} max_hard_limit={j['max_hard_limit']} "
+                f"min_soft_limit={j['min_soft_limit']} max_soft_limit={j['max_soft_limit']}\n")
+        sys.stderr.write(
+            f"DEBUG axis_mask={stat.axis_mask} kinematics_type={stat.kinematics_type} "
+            f"motion_mode={stat.motion_mode} interp_state={stat.interp_state} "
+            f"estop={stat.estop} enabled={stat.enabled} homed_all={stat.homed}\n")
+    except Exception as e:
+        sys.stderr.write(f"DEBUG joint dump failed: {e}\n")
+    import subprocess
+    for args in (["halcmd", "show", "pin", "halui.machine"],
+                 ["halcmd", "show", "pin", "joint.0"],
+                 ["halcmd", "show", "param", "joint.0"],
+                 ["halcmd", "show", "sig"]):
+        try:
+            out = subprocess.check_output(
+                args, stderr=subprocess.STDOUT, timeout=5).decode()
+            sys.stderr.write(f"DEBUG {' '.join(args)}:\n{out}\n")
+        except Exception as e:
+            sys.stderr.write(f"DEBUG {' '.join(args)} failed: {e}\n")
+    qlog = os.path.expanduser("~/qtdragon.log")
+    if os.path.exists(qlog):
+        try:
+            with open(qlog) as f:
+                tail = f.readlines()[-50:]
+            sys.stderr.write(f"DEBUG qtdragon.log tail:\n{''.join(tail)}\n")
+        except Exception as e:
+            sys.stderr.write(f"DEBUG read {qlog} failed: {e}\n")
     return False
 
 

tests/ui-smoke/_lib/launch.sh

@@ -56,6 +56,58 @@ export GST_PLUGIN_FEATURE_RANK="pulsesink:NONE,alsasink:NONE,osssink:NONE,oss4si
 export PULSE_SERVER=/dev/null
 export SDL_AUDIODRIVER=dummy
 
+# DEBUG #4054 clang CI bisect: PYTHONFAULTHANDLER was a candidate masker
+# of the qtvcp:481 Abort/Segfault. Disabled here for the test.
+# export PYTHONFAULTHANDLER=1
+# DEBUG #4054 clang CI: enable cores + glibc abort verbosity.
+ulimit -c unlimited
+export LIBC_FATAL_STDERR_=1
+
+# DEBUG #4054 clang CI: wrap qtvcp under gdb --batch so a SIGABRT/SIGSEGV
+# inside the Qt event loop (which faulthandler confirms is happening at
+# qtvcp.py:481 APP.exec()) gets a C-level backtrace. The wrapper is a
+# temp dir prepended to PATH that shadows qtvcp; the inner gdb invocation
+# uses the absolute path captured BEFORE shadowing so it does not recurse.
+# Diagnostics go to linuxcnc.err via stderr.
+REAL_QTVCP_PATH="$(command -v qtvcp || true)"
+if ! command -v gdb >/dev/null 2>&1; then
+    echo "DEBUG GDBWRAP installing gdb via apt" >&2
+    sudo apt-get install -y --no-install-recommends gdb 2>&1 | tail -3 >&2 || true
+fi
+echo "DEBUG GDBWRAP REAL_QTVCP_PATH=$REAL_QTVCP_PATH" >&2
+echo "DEBUG GDBWRAP have_gdb=$(command -v gdb || echo NO)" >&2
+if [ -z "$REAL_QTVCP_PATH" ]; then
+    echo "DEBUG GDBWRAP skipped: qtvcp not found on PATH=$PATH" >&2
+elif ! command -v gdb >/dev/null 2>&1; then
+    echo "DEBUG GDBWRAP skipped: gdb missing" >&2
+else
+    GDB_WRAP_DIR="$(mktemp -d -t qtvcp-gdb.XXXXXX)"
+    cat >"$GDB_WRAP_DIR/qtvcp" <<WRAP
+#!/bin/bash
+echo "DEBUG GDBWRAP active, running qtvcp under gdb (args: \$*)" >&2
+# Disable Python faulthandler so gdb's ptrace handler sees the signal
+# first instead of faulthandler converting it to a Python-only stack.
+unset PYTHONFAULTHANDLER
+exec gdb -batch -nx \\
+    -ex 'set pagination off' \\
+    -ex 'handle SIG33 nostop noprint pass' \\
+    -ex 'handle SIGCHLD nostop noprint pass' \\
+    -ex 'handle SIGPIPE nostop noprint pass' \\
+    -ex 'handle SIGABRT stop print nopass' \\
+    -ex 'handle SIGSEGV stop print nopass' \\
+    -ex run \\
+    -ex 'echo \n=== signal caught, dumping all-thread backtrace ===\n' \\
+    -ex 'thread apply all bt' \\
+    -ex 'echo \n=== source context of current frame ===\n' \\
+    -ex 'frame' \\
+    -ex 'list' \\
+    --args /usr/bin/python3 "$REAL_QTVCP_PATH" "\$@"
+WRAP
+    chmod +x "$GDB_WRAP_DIR/qtvcp"
+    export PATH="$GDB_WRAP_DIR:$PATH"
+    echo "DEBUG GDBWRAP installed at $GDB_WRAP_DIR, PATH now=$PATH" >&2
+fi
+
 # Export the per-invocation values so the inner bash -c receives them
 # as proper env vars (avoids embedding paths into the inner script
 # via quoting, which breaks on apostrophes / spaces).