Skip to content

Commit a468454

Browse files
abueideclaude
abueide
and
claude
committed
feat(tests): add crash detection, soak testing, and log capture to E2E suites
All verify processes now include a 10s soak period, device log capture to reports/logs/, and error pattern scanning for native crashes and RN JS errors. Cleanup processes capture final logs before teardown. The test framework summary lists diagnostic log files on failure. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent a20d93d commit a468454

6 files changed

Lines changed: 462 additions & 117 deletions

File tree

‎examples/android/tests/test-suite.yaml‎

Lines changed: 52 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -238,24 +238,56 @@ processes:
238238
while [ $attempt -lt $max_attempts ]; do
239239
if android.sh app status; then
240240
echo "App is running (verified after ${attempt}s)"
241-
# Save logcat snapshot on success too
242-
adb -s "$serial" logcat -d > "$_logs_dir/logcat-deploy-success.txt" 2>&1 || true
243-
echo "pass" > "$_step_dir/$_step.status"
244-
exit 0
241+
break
245242
fi
246243
attempt=$((attempt + 1))
247244
echo " Attempt $attempt/$max_attempts..."
248245
sleep 2
249246
done
250247
251-
# App not stable after deploy
252-
echo ""
253-
echo "=== App not stable after deploy - capturing diagnostics ==="
254-
adb -s "$serial" logcat -d -s AndroidRuntime:E 2>&1 | tail -30 || true
255-
adb -s "$serial" logcat -d > "$_logs_dir/logcat-deploy-unstable.txt" 2>&1 || true
248+
if [ $attempt -ge $max_attempts ]; then
249+
echo ""
250+
echo "=== App not stable after deploy - capturing diagnostics ==="
251+
adb -s "$serial" logcat -d -s AndroidRuntime:E 2>&1 | tail -30 || true
252+
adb -s "$serial" logcat -d > "$_logs_dir/logcat-deploy-unstable.txt" 2>&1 || true
253+
echo "Logcat saved to: $_logs_dir/logcat-deploy-unstable.txt"
254+
printf 'fail\nApp did not stay running within timeout\n' > "$_step_dir/$_step.status"
255+
exit 1
256+
fi
257+
258+
# Soak period — verify app survives for 10 seconds
259+
echo "App process confirmed, starting soak period (10s)..."
260+
sleep 10
261+
262+
if ! android.sh app status; then
263+
echo "FAIL: App crashed during soak period"
264+
adb -s "$serial" logcat -d > "$_logs_dir/logcat-soak-crash.txt" 2>&1 || true
265+
adb -s "$serial" logcat -d -s AndroidRuntime:E > "$_logs_dir/logcat-crash-fatal.txt" 2>&1 || true
266+
echo "Logcat saved to: $_logs_dir/logcat-soak-crash.txt"
267+
echo "Crash logs: $_logs_dir/logcat-crash-fatal.txt"
268+
printf 'fail\nApp crashed during soak period\n' > "$_step_dir/$_step.status"
269+
exit 1
270+
fi
271+
272+
# Always save logcat snapshot for analysis
273+
adb -s "$serial" logcat -d > "$_logs_dir/logcat-final.txt" 2>&1 || true
256274
257-
printf 'fail\nApp did not stay running within timeout\n' > "$_step_dir/$_step.status"
258-
exit 1
275+
# Scan for crash patterns in logcat
276+
crash_patterns="FATAL EXCEPTION|AndroidRuntime.*FATAL|Process.*crash"
277+
if grep -qiE "$crash_patterns" "$_logs_dir/logcat-final.txt" 2>/dev/null; then
278+
echo ""
279+
echo "FAIL: Crash patterns detected in logcat"
280+
echo "--- Error lines ---"
281+
grep -iE "$crash_patterns" "$_logs_dir/logcat-final.txt" | tail -20
282+
echo "---"
283+
echo "Full logcat: $_logs_dir/logcat-final.txt"
284+
printf 'fail\nCrash patterns detected in logcat\n' > "$_step_dir/$_step.status"
285+
exit 1
286+
fi
287+
288+
echo "pass" > "$_step_dir/$_step.status"
289+
echo "App verified healthy (process alive, no crashes detected)"
290+
echo "Logcat saved to: $_logs_dir/logcat-final.txt"
259291
depends_on:
260292
build-app:
261293
condition: process_completed_successfully
@@ -264,9 +296,17 @@ processes:
264296
availability:
265297
restart: "no"
266298

267-
# Cleanup - stop everything in pure mode, keep running in dev mode
299+
# Cleanup - capture final logs, then stop everything in pure mode
268300
cleanup-app:
269301
command: |
302+
_logs_dir="reports/logs"
303+
mkdir -p "$_logs_dir"
304+
305+
# Capture final logcat before teardown
306+
serial="${ANDROID_EMULATOR_SERIAL:-emulator-${EMU_PORT:-5554}}"
307+
adb -s "$serial" logcat -d > "$_logs_dir/logcat-cleanup.txt" 2>&1 || true
308+
echo "Final logcat: $_logs_dir/logcat-cleanup.txt"
309+
270310
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
271311
echo "Cleaning up (pure mode)..."
272312
android.sh app stop || true

‎examples/ios/tests/test-suite.yaml‎

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -161,31 +161,82 @@ processes:
161161
availability:
162162
restart: "no"
163163

164-
# Phase 5: Verify app is running (liveness check)
164+
# Phase 5: Verify app is running, healthy, and survives soak period
165165
verify-app-running:
166166
command: |
167167
set -e
168168
_step="verify-app"
169169
_step_dir="reports/steps"
170-
mkdir -p "$_step_dir"
170+
_logs_dir="reports/logs"
171+
mkdir -p "$_step_dir" "$_logs_dir"
171172
172-
if ios.sh app status; then
173-
echo "App is running"
174-
echo "pass" > "$_step_dir/$_step.status"
175-
exit 0
176-
else
173+
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/ios-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-ios-e2e}/simulator-udid.txt" 2>/dev/null || true)
174+
175+
# Initial liveness check
176+
if ! ios.sh app status; then
177+
echo "FAIL: App is not running"
178+
if [ -n "$UDID" ]; then
179+
xcrun simctl spawn "$UDID" log show --predicate 'eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate"' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
180+
echo "Simulator crash logs: $_logs_dir/simulator-crash.log"
181+
fi
177182
printf 'fail\nApp not found in running processes\n' > "$_step_dir/$_step.status"
178183
exit 1
179184
fi
185+
186+
# Soak period — verify app survives for 10 seconds
187+
echo "App process is running, starting soak period (10s)..."
188+
sleep 10
189+
190+
if ! ios.sh app status; then
191+
echo "FAIL: App crashed during soak period"
192+
if [ -n "$UDID" ]; then
193+
xcrun simctl spawn "$UDID" log show --predicate 'eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate"' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
194+
echo "Simulator crash logs: $_logs_dir/simulator-crash.log"
195+
fi
196+
printf 'fail\nApp crashed during soak period\n' > "$_step_dir/$_step.status"
197+
exit 1
198+
fi
199+
200+
# Capture simulator logs for analysis
201+
if [ -n "$UDID" ]; then
202+
xcrun simctl spawn "$UDID" log show --last 2m --style compact > "$_logs_dir/simulator-app.log" 2>&1 || true
203+
204+
# Scan for native crash patterns
205+
error_patterns="Terminating app|SIGABRT|EXC_BAD_ACCESS|EXC_CRASH|Fatal error|Assertion failure"
206+
if grep -qiE "$error_patterns" "$_logs_dir/simulator-app.log" 2>/dev/null; then
207+
echo ""
208+
echo "FAIL: Crash/error patterns detected in simulator logs"
209+
echo "--- Error lines ---"
210+
grep -iE "$error_patterns" "$_logs_dir/simulator-app.log" | head -20
211+
echo "---"
212+
echo "Full logs: $_logs_dir/simulator-app.log"
213+
printf 'fail\nCrash patterns detected in simulator logs\n' > "$_step_dir/$_step.status"
214+
exit 1
215+
fi
216+
fi
217+
218+
echo "pass" > "$_step_dir/$_step.status"
219+
echo "App verified healthy (process alive, no crashes detected)"
220+
[ -f "$_logs_dir/simulator-app.log" ] && echo "App logs: $_logs_dir/simulator-app.log"
180221
depends_on:
181222
deploy-app:
182223
condition: process_completed_successfully
183224
availability:
184225
restart: "no"
185226

186-
# Cleanup - stop everything in pure mode, keep running in dev mode
227+
# Cleanup - capture final logs, then stop everything in pure mode
187228
cleanup:
188229
command: |
230+
_logs_dir="reports/logs"
231+
mkdir -p "$_logs_dir"
232+
233+
# Capture final simulator logs before teardown
234+
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/ios-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-ios-e2e}/simulator-udid.txt" 2>/dev/null || true)
235+
if [ -n "$UDID" ]; then
236+
xcrun simctl spawn "$UDID" log show --last 5m --style compact > "$_logs_dir/simulator-final.log" 2>&1 || true
237+
echo "Final simulator logs: $_logs_dir/simulator-final.log"
238+
fi
239+
189240
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
190241
echo "Cleaning up (pure mode)..."
191242
ios.sh app stop || true

0 commit comments

Comments
 (0)