Skip to content

Commit d700bb1

Browse files
abueideclaude
abueide
and
claude
committed
fix(tests): use marker-file approach for reliable failure propagation
process-compose exits 0 even when processes fail if the summary process (the last to run) succeeds or is blocked. This caused CI jobs to pass despite actual test failures. Changes: - All test suites now use marker-file (.e2e-passed) to track success - Verification steps create marker only on actual success - test-summary.sh checks for marker and exits 1 if missing - Removed pkill usage (unavailable in --pure mode) - Removed metro-bundler process_healthy deps from cleanup (blocked cleanup when metro never started, preventing summary from running) - Fixed iOS build.sh scheme detection to use POSIX parameter expansion instead of BSD-incompatible sed alternation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 24a6e39 commit d700bb1

10 files changed

Lines changed: 140 additions & 87 deletions

File tree

‎examples/android/tests/test-suite.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,8 @@ processes:
137137
while [ $attempt -lt $max_attempts ]; do
138138
if adb -s $serial shell pidof "$app_id" >/dev/null 2>&1; then
139139
echo "✓ App process running (PID: $(adb -s $serial shell pidof "$app_id"))"
140+
mkdir -p reports
141+
touch reports/.e2e-passed
140142
exit 0
141143
fi
142144
attempt=$((attempt + 1))
@@ -179,16 +181,14 @@ processes:
179181
fi
180182
depends_on:
181183
run-app:
182-
condition: process_completed_successfully
184+
condition: process_completed
183185
availability:
184186
restart: "no"
185187

186-
# Summary - only runs when all test phases succeed
188+
# Summary - always runs, checks marker file for pass/fail
187189
summary:
188-
command: "bash tests/test-summary.sh 'Android E2E Test Suite' 'reports/android-e2e-logs'"
190+
command: "bash tests/test-summary.sh 'Android E2E Test Suite' 'reports/android-e2e-logs' 'reports/.e2e-passed'"
189191
depends_on:
190-
run-app:
191-
condition: process_completed_successfully
192192
cleanup-app:
193193
condition: process_completed
194194
availability:

‎examples/android/tests/test-summary.sh‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
#!/usr/bin/env bash
22
# Shared test summary script for process-compose test suites
3-
# Usage: test-summary.sh "Test Suite Name" "path/to/logs"
3+
# Usage: test-summary.sh "Test Suite Name" "path/to/logs" [marker_file]
4+
#
5+
# If marker_file is provided, checks for its existence to determine pass/fail.
6+
# This ensures process-compose exits non-zero when tests fail.
47

58
set -euo pipefail
69

710
SUITE_NAME="${1:-Test Suite}"
811
LOG_PATH="${2:-reports/logs}"
12+
MARKER_FILE="${3:-reports/.e2e-passed}"
913

1014
echo ""
1115
echo "===================================="
@@ -15,8 +19,22 @@ echo ""
1519
echo "Test Logs:"
1620
echo " ${LOG_PATH}"
1721
echo ""
18-
echo "All tests passed!"
19-
echo "===================================="
22+
23+
if [ -f "$MARKER_FILE" ]; then
24+
echo "All tests passed!"
25+
echo "===================================="
26+
rm -f "$MARKER_FILE"
27+
else
28+
echo "FAILED: Tests did not complete successfully"
29+
echo "===================================="
30+
# If TUI is enabled, sleep to keep results visible before failing
31+
if [ "${TEST_TUI:-false}" = "true" ] || [ "${TEST_TUI:-false}" = "1" ]; then
32+
echo ""
33+
echo "TUI mode: Waiting 30 seconds before exit (Ctrl+C to exit now)..."
34+
sleep 30
35+
fi
36+
exit 1
37+
fi
2038

2139
# If TUI is enabled, sleep to keep results visible
2240
if [ "${TEST_TUI:-false}" = "true" ] || [ "${TEST_TUI:-false}" = "1" ]; then

‎examples/ios/tests/test-suite.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ processes:
146146
echo "Verifying app is running on simulator: $DEVICE_UDID"
147147
if xcrun simctl spawn "$DEVICE_UDID" launchctl list | grep -q "${IOS_APP_BUNDLE_ID}"; then
148148
echo "✓ App is running: ${IOS_APP_BUNDLE_ID}"
149+
mkdir -p reports
150+
touch reports/.e2e-passed
149151
exit 0
150152
else
151153
echo "ERROR: App not found in running processes"
@@ -197,12 +199,10 @@ processes:
197199
availability:
198200
restart: "no"
199201

200-
# Summary - only runs when all test phases succeed
202+
# Summary - always runs, checks marker file for pass/fail
201203
summary:
202-
command: "bash tests/test-summary.sh 'iOS E2E Test Suite' 'reports/ios-e2e-logs'"
204+
command: "bash tests/test-summary.sh 'iOS E2E Test Suite' 'reports/ios-e2e-logs' 'reports/.e2e-passed'"
203205
depends_on:
204-
verify-app-running:
205-
condition: process_completed_successfully
206206
cleanup:
207207
condition: process_completed
208208
availability:

‎examples/ios/tests/test-summary.sh‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
#!/usr/bin/env bash
22
# Shared test summary script for process-compose test suites
3-
# Usage: test-summary.sh "Test Suite Name" "path/to/logs"
3+
# Usage: test-summary.sh "Test Suite Name" "path/to/logs" [marker_file]
4+
#
5+
# If marker_file is provided, checks for its existence to determine pass/fail.
6+
# This ensures process-compose exits non-zero when tests fail.
47

58
set -euo pipefail
69

710
SUITE_NAME="${1:-Test Suite}"
811
LOG_PATH="${2:-reports/logs}"
12+
MARKER_FILE="${3:-reports/.e2e-passed}"
913

1014
echo ""
1115
echo "===================================="
@@ -15,8 +19,22 @@ echo ""
1519
echo "Test Logs:"
1620
echo " ${LOG_PATH}"
1721
echo ""
18-
echo "All tests passed!"
19-
echo "===================================="
22+
23+
if [ -f "$MARKER_FILE" ]; then
24+
echo "All tests passed!"
25+
echo "===================================="
26+
rm -f "$MARKER_FILE"
27+
else
28+
echo "FAILED: Tests did not complete successfully"
29+
echo "===================================="
30+
# If TUI is enabled, sleep to keep results visible before failing
31+
if [ "${TEST_TUI:-false}" = "true" ] || [ "${TEST_TUI:-false}" = "1" ]; then
32+
echo ""
33+
echo "TUI mode: Waiting 30 seconds before exit (Ctrl+C to exit now)..."
34+
sleep 30
35+
fi
36+
exit 1
37+
fi
2038

2139
# If TUI is enabled, sleep to keep results visible
2240
if [ "${TEST_TUI:-false}" = "true" ] || [ "${TEST_TUI:-false}" = "1" ]; then

‎examples/react-native/tests/test-suite-all-e2e.yaml‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,8 @@ processes:
279279
while [ $attempt -lt $max_attempts ]; do
280280
if adb -s $serial shell pidof "$app_id" >/dev/null 2>&1; then
281281
echo "✓ App process running (PID: $(adb -s $serial shell pidof "$app_id"))"
282+
mkdir -p reports
283+
touch reports/.e2e-android-passed
282284
exit 0
283285
fi
284286
attempt=$((attempt + 1))
@@ -309,6 +311,8 @@ processes:
309311
while [ $attempt -lt $max_attempts ]; do
310312
if xcrun simctl spawn "$DEVICE_UDID" launchctl list | grep -q "${IOS_APP_BUNDLE_ID}"; then
311313
echo "✓ App is running: ${IOS_APP_BUNDLE_ID}"
314+
mkdir -p reports
315+
touch reports/.e2e-ios-passed
312316
exit 0
313317
fi
314318
attempt=$((attempt + 1))
@@ -324,16 +328,27 @@ processes:
324328
availability:
325329
restart: "no"
326330

331+
# Mark passed - only runs when both platforms verified successfully
332+
mark-passed:
333+
command: |
334+
mkdir -p reports
335+
touch reports/.e2e-passed
336+
echo "✓ All platforms verified successfully"
337+
depends_on:
338+
verify-android-running:
339+
condition: process_completed_successfully
340+
verify-ios-running:
341+
condition: process_completed_successfully
342+
availability:
343+
restart: "no"
344+
327345
# Cleanup - runs when both platforms complete
328346
cleanup:
329347
command: |
330348
# Stop Metro bundler explicitly
331349
echo "Stopping Metro bundler..."
332350
metro.sh stop all || true
333351
334-
# Also kill the metro.sh wrapper process
335-
pkill -f "metro.sh start all" || true
336-
337352
if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
338353
echo "🧹 Cleaning up (pure mode)..."
339354
@@ -361,23 +376,15 @@ processes:
361376
echo "✓ Test complete - apps and emulator/simulator still running"
362377
fi
363378
depends_on:
364-
verify-android-running:
365-
condition: process_completed
366-
verify-ios-running:
379+
mark-passed:
367380
condition: process_completed
368-
metro-bundler:
369-
condition: process_healthy
370381
availability:
371382
restart: "no"
372383

373-
# Summary - only runs when all test phases succeed
384+
# Summary - always runs, checks marker file for pass/fail
374385
summary:
375-
command: "bash tests/test-summary.sh 'React Native All Platforms E2E Test Suite' 'reports/react-native-all-e2e-logs'"
386+
command: "bash tests/test-summary.sh 'React Native All Platforms E2E Test Suite' 'reports/react-native-all-e2e-logs' 'reports/.e2e-passed'"
376387
depends_on:
377-
verify-android-running:
378-
condition: process_completed_successfully
379-
verify-ios-running:
380-
condition: process_completed_successfully
381388
cleanup:
382389
condition: process_completed
383390
availability:

‎examples/react-native/tests/test-suite-android-e2e.yaml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,8 @@ processes:
185185
while [ $attempt -lt $max_attempts ]; do
186186
if adb -s $serial shell pidof "$app_id" >/dev/null 2>&1; then
187187
echo "✓ App process running (PID: $(adb -s $serial shell pidof "$app_id"))"
188+
mkdir -p reports
189+
touch reports/.e2e-passed
188190
exit 0
189191
fi
190192
attempt=$((attempt + 1))
@@ -211,9 +213,6 @@ processes:
211213
echo "Stopping Metro bundler..."
212214
metro.sh stop android || true
213215
214-
# Also kill the metro.sh wrapper process
215-
pkill -f "metro.sh start android" || true
216-
217216
if [ "${IN_NIX_SHELL:-}" = "pure" ]; then
218217
echo "🧹 Cleaning up (pure mode)..."
219218
app_id="${ANDROID_APP_ID:-$(cat ${ANDROID_RUNTIME_DIR}/app-id.txt 2>/dev/null || true)}"
@@ -226,17 +225,13 @@ processes:
226225
depends_on:
227226
verify-app-running:
228227
condition: process_completed
229-
metro-bundler:
230-
condition: process_healthy
231228
availability:
232229
restart: "no"
233230

234-
# Summary - only runs when all test phases succeed
231+
# Summary - always runs, checks marker file for pass/fail
235232
summary:
236-
command: "bash tests/test-summary.sh 'React Native Android E2E Test Suite' 'reports/react-native-android-e2e-logs'"
233+
command: "bash tests/test-summary.sh 'React Native Android E2E Test Suite' 'reports/react-native-android-e2e-logs' 'reports/.e2e-passed'"
237234
depends_on:
238-
verify-app-running:
239-
condition: process_completed_successfully
240235
cleanup:
241236
condition: process_completed
242237
availability:

‎examples/react-native/tests/test-suite-ios-e2e.yaml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,8 @@ processes:
219219
while [ $attempt -lt $max_attempts ]; do
220220
if xcrun simctl spawn "$DEVICE_UDID" launchctl list | grep -q "${IOS_APP_BUNDLE_ID}"; then
221221
echo "✓ App is running: ${IOS_APP_BUNDLE_ID}"
222+
mkdir -p reports
223+
touch reports/.e2e-passed
222224
exit 0
223225
fi
224226
attempt=$((attempt + 1))
@@ -241,9 +243,6 @@ processes:
241243
echo "Stopping Metro bundler..."
242244
metro.sh stop ios || true
243245
244-
# Also kill the metro.sh wrapper process
245-
pkill -f "metro.sh start ios" || true
246-
247246
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
248247
echo "🧹 Cleaning up (pure mode)..."
249248
@@ -268,17 +267,13 @@ processes:
268267
depends_on:
269268
verify-app-running:
270269
condition: process_completed
271-
metro-bundler:
272-
condition: process_healthy
273270
availability:
274271
restart: "no"
275272

276-
# Summary - only runs when all test phases succeed
273+
# Summary - always runs, checks marker file for pass/fail
277274
summary:
278-
command: "bash tests/test-summary.sh 'React Native iOS E2E Test Suite' 'reports/react-native-ios-e2e-logs'"
275+
command: "bash tests/test-summary.sh 'React Native iOS E2E Test Suite' 'reports/react-native-ios-e2e-logs' 'reports/.e2e-passed'"
279276
depends_on:
280-
verify-app-running:
281-
condition: process_completed_successfully
282277
cleanup:
283278
condition: process_completed
284279
availability:

‎examples/react-native/tests/test-suite-web-e2e.yaml‎

Lines changed: 33 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -69,65 +69,62 @@ processes:
6969
timeout_seconds: 60
7070
success_threshold: 1
7171

72-
# Phase 4: Open browser with web bundle
73-
open-browser:
72+
# Phase 4: Verify web app is being served
73+
verify-web-serving:
7474
command: |
7575
# Source Metro environment to get correct port
7676
. ${REACT_NATIVE_VIRTENV}/metro/env-web.sh
7777
78-
echo "🌐 Opening browser to Metro web bundle..."
79-
echo " URL: http://localhost:$METRO_PORT"
80-
81-
# Open default browser
82-
if command -v open >/dev/null 2>&1; then
83-
# macOS
84-
open "http://localhost:$METRO_PORT"
85-
elif command -v xdg-open >/dev/null 2>&1; then
86-
# Linux
87-
xdg-open "http://localhost:$METRO_PORT"
88-
elif command -v start >/dev/null 2>&1; then
89-
# Windows (Git Bash/WSL)
90-
start "http://localhost:$METRO_PORT"
91-
else
92-
echo "⚠ Could not detect browser launcher"
93-
echo " Please open: http://localhost:$METRO_PORT"
94-
fi
95-
96-
echo "✓ Browser launched"
97-
echo ""
98-
echo "Metro is running at: http://localhost:$METRO_PORT"
99-
echo "Press Ctrl+C to stop Metro and exit"
78+
echo "Verifying web app is being served on port $METRO_PORT..."
79+
80+
# Use node to make HTTP request (curl may not be available in --pure)
81+
node -e "
82+
const http = require('http');
83+
const url = 'http://localhost:' + process.env.METRO_PORT;
84+
http.get(url, (res) => {
85+
let data = '';
86+
res.on('data', (chunk) => data += chunk);
87+
res.on('end', () => {
88+
if (res.statusCode === 200) {
89+
console.log('✓ Web app is being served (HTTP ' + res.statusCode + ')');
90+
console.log(' Response size: ' + data.length + ' bytes');
91+
require('fs').mkdirSync('reports', { recursive: true });
92+
require('fs').writeFileSync('reports/.e2e-passed', '');
93+
process.exit(0);
94+
} else {
95+
console.error('ERROR: Unexpected status code: ' + res.statusCode);
96+
process.exit(1);
97+
}
98+
});
99+
}).on('error', (err) => {
100+
console.error('ERROR: Could not connect to Metro: ' + err.message);
101+
process.exit(1);
102+
});
103+
"
100104
depends_on:
101105
metro-bundler:
102106
condition: process_healthy
107+
build-web:
108+
condition: process_completed_successfully
103109
availability:
104110
restart: "no"
105111

106112
# Cleanup
107113
cleanup:
108114
command: |
109-
# Stop Metro bundler explicitly
110115
echo "Stopping Metro bundler..."
111116
metro.sh stop web || true
112-
113-
# Also kill the metro.sh wrapper process
114-
pkill -f "metro.sh start web" || true
115-
116117
echo "✓ Test complete"
117118
depends_on:
118-
open-browser:
119+
verify-web-serving:
119120
condition: process_completed
120-
metro-bundler:
121-
condition: process_healthy
122121
availability:
123122
restart: "no"
124123

125-
# Summary - only runs when all test phases succeed
124+
# Summary - always runs, checks marker file for pass/fail
126125
summary:
127-
command: "bash tests/test-summary.sh 'React Native Web E2E Test Suite' 'reports/react-native-web-e2e-logs'"
126+
command: "bash tests/test-summary.sh 'React Native Web E2E Test Suite' 'reports/react-native-web-e2e-logs' 'reports/.e2e-passed'"
128127
depends_on:
129-
open-browser:
130-
condition: process_completed_successfully
131128
cleanup:
132129
condition: process_completed
133130
availability:

0 commit comments

Comments
 (0)