Skip to content

Commit 16004e0

Browse files
antoyoantoyo
committed
Add CI debug workflow for stack-overflow detection failure
Temporary debug branch: runs the two failing stack-overflow UI tests on Ubuntu CI and, on failure, dumps an automated diagnostic (strace of sigaltstack/clone per thread, gdb trace of make_handler/set_current_info, objdump of make_handler) plus opens a tmate SSH session.
1 parent d017855 commit 16004e0

2 files changed

Lines changed: 136 additions & 0 deletions

File tree

‎.github/debug_so.sh‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
#!/usr/bin/env bash
2+
# Automated diagnostic for the stack-overflow detection miscompilation.
3+
# Runs in CI on failure; everything is printed to the job log.
4+
set +e
5+
6+
SYSROOT="$PWD/build/build_sysroot/sysroot"
7+
STDLIB="$SYSROOT/lib/rustlib/x86_64-unknown-linux-gnu/lib"
8+
export LD_LIBRARY_PATH="/usr/lib:$STDLIB"
9+
ulimit -c unlimited 2>/dev/null
10+
11+
echo "===================== ENV ====================="
12+
uname -a
13+
ldd --version | head -1
14+
echo "STDLIB=$STDLIB"
15+
LIB=$(ls "$STDLIB"/libstd*.so 2>/dev/null | head -1)
16+
echo "libstd.so: $LIB size=$(stat -c%s "$LIB" 2>/dev/null)"
17+
18+
echo "===================== LOCATE BINARIES ====================="
19+
OO=$(find build/rust -type f -path '*test/ui/runtime/out-of-stack/a' 2>/dev/null | head -1)
20+
SP=$(find build/rust -type f -path '*test/ui/abi/stack-probes*/a' 2>/dev/null | head -1)
21+
echo "out-of-stack binary: ${OO:-<none>}"
22+
echo "stack-probes binary: ${SP:-<none>}"
23+
24+
debug_one() {
25+
local BIN="$1" ARG="$2"
26+
[ -z "$BIN" ] && return
27+
echo ""
28+
echo "############################################################"
29+
echo "# DEBUG $BIN $ARG"
30+
echo "############################################################"
31+
32+
echo "----- reproduce (expect 139=SIGSEGV/undetected-bug, 134=SIGABRT/detected-ok) -----"
33+
"$BIN" "$ARG"; echo "exit=$?"
34+
35+
echo "----- strace: clone + sigaltstack + SIGSEGV/BUS sigaction (per-thread) -----"
36+
# Hypothesis: the SPAWNED thread never completes make_handler, so it never
37+
# calls sigaltstack. In a working run each new thread issues sigaltstack.
38+
strace -f -e trace=clone,clone3,sigaltstack,rt_sigaction "$BIN" "$ARG" 2>strace.txt
39+
grep -nE "clone|sigaltstack|rt_sigaction\((SIGSEGV|SIGBUS)" strace.txt
40+
echo "sigaltstack total: $(grep -c 'sigaltstack(' strace.txt) clone total: $(grep -cE 'clone[3]?\(' strace.txt)"
41+
42+
echo "----- gdb: is make_handler / set_current_info reached by the SPAWNED thread? -----"
43+
gdb -q -batch -nx \
44+
-ex 'set pagination off' -ex 'set confirm off' \
45+
-ex 'rbreak stack_overflow.*imp.*make_handler' \
46+
-ex 'rbreak stack_overflow.*thread_info.*set_current_info' \
47+
-ex "run $ARG" \
48+
-ex 'printf "== HIT %s (thread %d) ==\n", "1", $_thread' -ex 'bt 4' -ex 'continue' \
49+
-ex 'printf "== HIT %s (thread %d) ==\n", "2", $_thread' -ex 'bt 4' -ex 'continue' \
50+
-ex 'printf "== HIT %s (thread %d) ==\n", "3", $_thread' -ex 'bt 4' -ex 'continue' \
51+
-ex 'printf "== HIT %s (thread %d) ==\n", "4", $_thread' -ex 'bt 4' -ex 'continue' \
52+
-ex 'printf "== HIT %s (thread %d) ==\n", "5", $_thread' -ex 'bt 4' -ex 'continue' \
53+
-ex 'printf "== reached program end / signal ==\n"' -ex 'info program' \
54+
--args "$BIN" "$ARG" 2>&1 \
55+
| grep -vE "Missing separate debuginfos|Download failed|Reading symbols|no debugging symbols" | head -160
56+
}
57+
58+
debug_one "$OO" "silent-thread"
59+
debug_one "$SP" "child-recurse"
60+
debug_one "$SP" "child-frame"
61+
62+
echo ""
63+
echo "===================== OBJDUMP make_handler (from failing libstd.so) ====================="
64+
objdump -d "$LIB" 2>/dev/null | awk '
65+
/stack_overflow.*imp.*make_handler>:/{p=1}
66+
p{print}
67+
p&&/\tret/{exit}' | head -90
68+
69+
echo ""
70+
echo "===================== NEED_ALTSTACK / set_current_info call sites ====================="
71+
objdump -d "$LIB" 2>/dev/null | grep -nE "NEED_ALTSTACK|call.*make_handler|call.*Handler.*new|call.*set_current_info" | head -20
72+
73+
echo "===================== DEBUG DONE ====================="

‎.github/workflows/debug-so.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Debug stack-overflow
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- debug/so-ci
8+
9+
permissions:
10+
contents: read
11+
12+
env:
13+
RUST_BACKTRACE: 1
14+
LLVM_BIN_DIR: /usr/bin
15+
16+
jobs:
17+
debug:
18+
runs-on: ubuntu-24.04
19+
steps:
20+
- uses: actions/checkout@v4
21+
22+
- name: Setup rust toolchain
23+
run: rustup show
24+
25+
- name: Setup rust cache
26+
uses: Swatinem/rust-cache@v2
27+
28+
- name: Install packages
29+
run: sudo apt-get install -y ninja-build ripgrep gdb strace
30+
31+
- name: Download artifact
32+
run: curl -LO https://github.com/rust-lang/gcc/releases/latest/download/gcc-15.deb
33+
34+
- name: Setup path to libgccjit
35+
run: |
36+
sudo dpkg --force-overwrite -i gcc-15.deb
37+
echo 'gcc-path = "/usr/lib/"' > config.toml
38+
39+
- name: Set default GCC to gcc-14
40+
run: sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-14 30
41+
42+
- name: Enable core dumps
43+
run: |
44+
ulimit -c unlimited
45+
sudo sysctl -w kernel.core_pattern=core.%p.%e || true
46+
47+
- name: Prepare
48+
run: ./y.sh prepare
49+
50+
- name: Build backend and sysroot
51+
run: ./y.sh build --release --sysroot
52+
53+
- name: Run the failing stack-overflow tests
54+
id: stacktests
55+
run: |
56+
./y.sh test --release --run-ui-tests -- \
57+
tests/ui/runtime/out-of-stack.rs \
58+
tests/ui/abi/stack-probes.rs \
59+
--force-rerun
60+
61+
- name: Automated debug dump
62+
if: ${{ failure() }}
63+
run: bash .github/debug_so.sh

0 commit comments

Comments
 (0)