Skip to content

Commit 8d32e87

Browse files
mikehardymikehardy
committed
test(ci, ios): let runner load settle, instrument simulator/host better
1 parent aa6b1f1 commit 8d32e87

6 files changed

Lines changed: 160 additions & 15 deletions

File tree

‎.github/workflows/scripts/boot-simulator.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,16 @@ wait_for_simulator_ready() {
155155
return 1
156156
}
157157

158+
BOOT_MODE="${RNFB_SIM_BOOT_MODE:-full}"
159+
160+
# shellcheck source=simulator-logging.sh
161+
source "$(dirname "$0")/simulator-logging.sh"
162+
163+
if [[ "$BOOT_MODE" == "logs" ]]; then
164+
restart_simulator_logging || true
165+
exit 0
166+
fi
167+
158168
# Get our simulator name from our test Detox config
159169
pushd "$(dirname "$0")/../../../tests" >/dev/null || exit 1
160170
SIM="$(grep iPhone .detoxrc.js | head -1 | cut -d"'" -f2)"
@@ -220,3 +230,7 @@ log_boot_status "install complete in $((SECONDS - install_start))s"
220230
popd >/dev/null || exit 1
221231

222232
log_boot_status "phase=complete device=\"${SIM}\" ready with test app installed"
233+
234+
if [[ "${RNFB_START_SIM_LOGS:-1}" == "1" && "$BOOT_MODE" == "full" ]]; then
235+
restart_simulator_logging || true
236+
fi

‎.github/workflows/scripts/flake-summary.sh‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ DETOX_LOG="${RNFB_DETOX_LOG:-}"
1919
fi
2020
echo "=== ${label}: ${file} ($(wc -l <"$file" | tr -d ' ') lines) ==="
2121
rg -n \
22-
'\[jet-ws\]|\[rnfb-e2e\]|\[jet-coverage\]|\[rnfb-lifecycle\]|RETRYABLE_DISCONNECT|reconnect_recovered|FrontBoard|FBSOpenApplication|coverage-ready|retry-eligibility|install-state|terminateApp|launchApp failure|Jet attempt|FAIL e2e' \
22+
'\[jet-ws\]|\[rnfb-e2e\]|\[jet-coverage\]|\[rnfb-lifecycle\]|RETRYABLE_DISCONNECT|reconnect_recovered|FrontBoard|FBSOpenApplication|coverage-ready|retry-eligibility|install-state|terminateApp|launchApp failure|Jet attempt|FAIL e2e|orchestrate-state|Child process terminated|log stream restarted|\[load-settle\]' \
2323
"$file" 2>/dev/null | tail -200 || true
2424
echo ""
2525
}
@@ -31,6 +31,24 @@ DETOX_LOG="${RNFB_DETOX_LOG:-}"
3131
summarize "resource-monitor" "resource-monitor.log"
3232
summarize "resource-monitor-android" "resource-monitor-android.log"
3333
summarize "metro-log" "metro.log"
34+
35+
echo "=== disconnect_context vs resource-monitor load ==="
36+
if [[ -f "${DETOX_LOG}" ]]; then
37+
rg -n '\[jet-ws\] disconnect_context' "${DETOX_LOG}" 2>/dev/null | tail -20 || true
38+
fi
39+
if [[ -f "resource-monitor.log" ]]; then
40+
echo "--- resource-monitor load around disconnect (last 80 load lines) ---"
41+
rg -n 'load averages|^\[resource-monitor\] ps-empty' resource-monitor.log 2>/dev/null | tail -80 || true
42+
fi
43+
echo ""
44+
45+
echo "=== log stream death markers ==="
46+
for f in simulator.log testing.log springboard-invertase.log; do
47+
if [[ -f "$f" ]]; then
48+
rg -n 'Child process terminated|log stream restarted' "$f" 2>/dev/null | tail -20 || true
49+
fi
50+
done
51+
echo ""
3452
} >"$OUT"
3553

3654
echo "[flake-summary] wrote ${OUT} ($(wc -l <"$OUT" | tr -d ' ') lines)"

‎.github/workflows/scripts/resource-monitor.sh‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,27 @@ while true; do
1313
echo "--- load ---"
1414
uptime
1515
echo "--- top (pid pcpu pmem rss command) ---"
16-
ps -arc www -o pid,pcpu,pmem,rss,etime,command 2>/dev/null | head -30 || true
16+
top_ps="$(ps -arc www -o pid,pcpu,pmem,rss,etime,command 2>/dev/null | head -30 || true)"
17+
if [[ -z "$(echo "$top_ps" | sed -n '2,$p' | grep -v '^[[:space:]]*$' | head -1)" ]]; then
18+
echo "[resource-monitor] ps-empty falling back to ps aux + top -l 1"
19+
ps aux 2>/dev/null | head -30 || true
20+
top -l 1 -n 10 -stats pid,cpu,mem,command 2>/dev/null | head -25 || true
21+
else
22+
echo "$top_ps"
23+
fi
1724
echo "--- e2e-related ---"
18-
ps -arc www -o pid,pcpu,pmem,rss,etime,command 2>/dev/null \
25+
e2e_ps="$(ps -arc www -o pid,pcpu,pmem,rss,etime,command 2>/dev/null \
1926
| grep -E 'testing|node |java |Simulator|Metro|simctl|log stream|screencapture' \
20-
| grep -v grep || true
27+
| grep -v grep || true)"
28+
if [[ -z "$e2e_ps" ]]; then
29+
echo "[resource-monitor] ps-empty falling back to ps aux grep"
30+
ps aux 2>/dev/null \
31+
| grep -E 'testing|node |java |Simulator|Metro|simctl|log stream|screencapture' \
32+
| grep -v grep \
33+
| head -20 || true
34+
else
35+
echo "$e2e_ps"
36+
fi
2137
echo ""
2238
} >>"$LOG_FILE" 2>&1
2339
sleep "$INTERVAL_SEC"

‎.github/workflows/scripts/simulator-logging.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
# Start or restart iOS simulator log streams and resource monitor (sourced by boot-simulator.sh).
3+
set -uo pipefail
4+
5+
restart_simulator_logging() {
6+
local with_stdout="${RNFB_SIM_LOG_STDOUT:-0}"
7+
local log_dir="${RNFB_SIM_LOG_DIR:-.}"
8+
local simulator_log="${log_dir}/simulator.log"
9+
local testing_log="${log_dir}/testing.log"
10+
local springboard_log="${log_dir}/springboard-invertase.log"
11+
local resource_log="${log_dir}/resource-monitor.log"
12+
local repo_root="${RNFB_REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}"
13+
14+
if ! xcrun simctl list devices booted 2>/dev/null | grep -q Booted; then
15+
echo "[boot-status] phase=log_streams skipped=no_booted_simulator"
16+
return 1
17+
fi
18+
19+
echo "[boot-status] phase=log_streams_restart ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) stdout=${with_stdout} dir=${log_dir}"
20+
21+
pkill -f 'simctl spawn booted log stream' 2>/dev/null || true
22+
pkill -f 'simctl io booted recordVideo' 2>/dev/null || true
23+
pkill -f resource-monitor.sh 2>/dev/null || true
24+
sleep 1
25+
26+
{
27+
echo ""
28+
echo "=== log stream restarted $(date -u +%Y-%m-%dT%H:%M:%SZ) ==="
29+
} >>"$simulator_log" 2>/dev/null || : >"$simulator_log"
30+
{
31+
echo ""
32+
echo "=== log stream restarted $(date -u +%Y-%m-%dT%H:%M:%SZ) ==="
33+
} >>"$testing_log" 2>/dev/null || : >"$testing_log"
34+
{
35+
echo ""
36+
echo "=== log stream restarted $(date -u +%Y-%m-%dT%H:%M:%SZ) ==="
37+
} >>"$springboard_log" 2>/dev/null || : >"$springboard_log"
38+
39+
if [[ "$with_stdout" == "1" ]]; then
40+
xcrun simctl io booted recordVideo --codec=h264 -f "${log_dir}/simulator.mp4" 2>&1 &
41+
xcrun simctl spawn booted log stream --level debug --style compact 2>&1 | tee -a "$simulator_log" &
42+
xcrun simctl spawn booted log stream --level debug --style compact \
43+
--predicate 'process == "testing"' 2>&1 | tee -a "$testing_log" &
44+
xcrun simctl spawn booted log stream --level debug --style compact \
45+
--predicate 'process == "SpringBoard" AND eventMessage CONTAINS "invertase"' 2>&1 | tee -a "$springboard_log" &
46+
else
47+
nohup sh -c "xcrun simctl io booted recordVideo --codec=h264 -f '${log_dir}/simulator.mp4' 2>&1 &" >/dev/null 2>&1 &
48+
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact >>'${simulator_log}' 2>&1 &" >/dev/null 2>&1 &
49+
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact --predicate 'process == \"testing\"' >>'${testing_log}' 2>&1 &" >/dev/null 2>&1 &
50+
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact --predicate 'process == \"SpringBoard\" AND eventMessage CONTAINS \"invertase\"' >>'${springboard_log}' 2>&1 &" >/dev/null 2>&1 &
51+
fi
52+
53+
chmod +x "${repo_root}/.github/workflows/scripts/resource-monitor.sh"
54+
RNFB_RESOURCE_MONITOR_LOG="$resource_log" nohup "${repo_root}/.github/workflows/scripts/resource-monitor.sh" >/dev/null 2>&1 &
55+
56+
echo "[boot-status] phase=log_streams_started video=${log_dir}/simulator.mp4 monitor=${resource_log}"
57+
}

‎.github/workflows/scripts/wait-for-load-settle.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
# Wait for macOS host load average to drop before starting Detox (experimental CI settle step).
3+
set -euo pipefail
4+
5+
MAX_WAIT_SEC="${RNFB_LOAD_SETTLE_MAX_WAIT_SEC:-1200}"
6+
POLL_SEC="${RNFB_LOAD_SETTLE_POLL_SEC:-5}"
7+
MAX_LOAD="${RNFB_LOAD_SETTLE_MAX_LOAD:-10}"
8+
9+
parse_load1() {
10+
# uptime: ... load averages: 1.23 4.56 7.89
11+
uptime 2>/dev/null | sed -n 's/.*load averages: \([0-9.]*\).*/\1/p' | head -1
12+
}
13+
14+
elapsed=0
15+
echo "[load-settle] max_wait=${MAX_WAIT_SEC}s poll=${POLL_SEC}s threshold=${MAX_LOAD}"
16+
17+
while (( elapsed <= MAX_WAIT_SEC )); do
18+
load1="$(parse_load1)"
19+
if [[ -z "$load1" ]]; then
20+
echo "[load-settle] ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) elapsed=${elapsed}s load1=unknown (could not parse uptime)"
21+
else
22+
echo "[load-settle] ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) elapsed=${elapsed}s load1=${load1} threshold=${MAX_LOAD}"
23+
if awk -v load="$load1" -v max="$MAX_LOAD" 'BEGIN { exit (load < max) ? 0 : 1 }'; then
24+
echo "[load-settle] host load ${load1} < ${MAX_LOAD} after ${elapsed}s — proceeding"
25+
exit 0
26+
fi
27+
fi
28+
29+
if (( elapsed >= MAX_WAIT_SEC )); then
30+
break
31+
fi
32+
33+
sleep "$POLL_SEC"
34+
elapsed=$((elapsed + POLL_SEC))
35+
done
36+
37+
echo "[load-settle] WARN: timed out after ${MAX_WAIT_SEC}s with load1=${load1:-unknown} (threshold=${MAX_LOAD}) — proceeding anyway"
38+
exit 0

‎.github/workflows/tests_e2e_ios.yml‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ jobs:
9494
runs-on: macos-26
9595
needs: matrix_prep
9696
# TODO matrix across APIs, at least 11 and 15 (lowest to highest)
97-
timeout-minutes: 87
97+
timeout-minutes: 107
9898
env:
9999
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
100100
CCACHE_SLOPPINESS: clang_index_store,file_stat_matches,include_file_ctime,include_file_mtime,ivfsoverlay,pch_defines,modules,system_headers,time_macros
@@ -342,22 +342,24 @@ jobs:
342342
timeout_minutes: 12
343343
retry_wait_seconds: 60
344344
max_attempts: 3
345-
command: ./.github/workflows/scripts/boot-simulator.sh
345+
command: RNFB_START_SIM_LOGS=0 ./.github/workflows/scripts/boot-simulator.sh
346346

347347
- name: Start Simulator Recordings and Log
348-
# Start after Pre-Boot so booted exists and logging covers the Detox run (not a fixed delay).
348+
# Log streams are owned by boot-simulator.sh (logs-only mode after pre-boot).
349349
continue-on-error: true
350350
run: |
351-
nohup sh -c "xcrun simctl io booted recordVideo --codec=h264 -f simulator.mp4 2>&1 &"
352-
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact > simulator.log 2>&1 &"
353-
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact --predicate 'process == \"testing\"' > testing.log 2>&1 &"
354-
nohup sh -c "xcrun simctl spawn booted log stream --level debug --style compact --predicate 'process == \"SpringBoard\" AND eventMessage CONTAINS \"invertase\"' > springboard-invertase.log 2>&1 &"
355-
chmod +x ./.github/workflows/scripts/resource-monitor.sh
356-
nohup ./.github/workflows/scripts/resource-monitor.sh &
351+
chmod +x ./.github/workflows/scripts/boot-simulator.sh
352+
RNFB_SIM_BOOT_MODE=logs ./.github/workflows/scripts/boot-simulator.sh
353+
354+
- name: Wait for host load to settle
355+
continue-on-error: true
356+
run: |
357+
chmod +x ./.github/workflows/scripts/wait-for-load-settle.sh
358+
./.github/workflows/scripts/wait-for-load-settle.sh
357359
358360
- name: Detox Test Debug
359361
if: contains(matrix.buildmode, 'debug')
360-
timeout-minutes: 62
362+
timeout-minutes: 82
361363
run: |
362364
yarn tests:ios:test-cover 2>&1 | tee detox-step.log
363365
exit ${PIPESTATUS[0]}
@@ -369,7 +371,7 @@ jobs:
369371

370372
- name: Detox Test Release
371373
if: contains(matrix.buildmode, 'release')
372-
timeout-minutes: 62
374+
timeout-minutes: 82
373375
run: |
374376
yarn tests:ios:test:release 2>&1 | tee detox-step.log
375377
exit ${PIPESTATUS[0]}

0 commit comments

Comments
 (0)