Skip to content

Commit e9afa46

Browse files
abueideclaude
andcommitted
feat(tests): add progressive summary generation and include resolution tests in test:fast
Add _regenerate_summary() to test-framework.sh so each test_summary() call writes per-suite JSON with timestamps and regenerates the combined reports/summary.md. This enables progressive report building as suites complete. Also adds android-apk-resolution and ios-app-resolution tests to test:fast since they're pure unit tests requiring no services. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f39b2a1 commit e9afa46

8 files changed

Lines changed: 168 additions & 110 deletions

File tree

devbox.json

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@
107107
"test:plugin:android:apk-detection": [
108108
"(cd examples/android && devbox run bash ../../plugins/tests/android/test-apk-detection.sh)"
109109
],
110+
"test:plugin:android:apk-resolution": [
111+
"bash plugins/tests/android/test-apk-resolution.sh"
112+
],
110113
"test:plugin:android:emulator-detection": [
111114
"(cd examples/android && devbox run bash ../../plugins/tests/android/test-emulator-detection.sh)"
112115
],
@@ -116,7 +119,8 @@
116119
"test:plugin:android": [
117120
"devbox run test:plugin:android:lib",
118121
"devbox run test:plugin:android:devices",
119-
"devbox run test:plugin:android:apk-detection"
122+
"devbox run test:plugin:android:apk-detection",
123+
"devbox run test:plugin:android:apk-resolution"
120124
],
121125
"test:plugin:android:all": [
122126
"echo 'Running all Android plugin tests (including emulator tests)...'",
@@ -131,9 +135,13 @@
131135
"test:plugin:ios:devices": [
132136
"sh plugins/tests/ios/test-devices.sh"
133137
],
138+
"test:plugin:ios:app-resolution": [
139+
"bash plugins/tests/ios/test-app-resolution.sh"
140+
],
134141
"test:plugin:ios": [
135142
"devbox run test:plugin:ios:lib",
136-
"devbox run test:plugin:ios:devices"
143+
"devbox run test:plugin:ios:devices",
144+
"devbox run test:plugin:ios:app-resolution"
137145
],
138146
"test:plugin:devbox-mcp": [
139147
"echo 'Running devbox-mcp plugin tests...'",

examples/android/devbox.d/android/devices/devices.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@
1414
}
1515
],
1616
"checksum": "8df4d3393b61fbbb08e45cf8762f95c521316938e514527916e4fce88a849d57",
17-
"generated_at": "2026-02-19T18:07:54Z"
17+
"generated_at": "2026-02-19T21:24:53Z"
1818
}

examples/ios/devbox.d/ios/devices/devices.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@
1010
}
1111
],
1212
"checksum": "4d5276f203d7ad62860bfc067f76194df53be449d4aa8a3b2d069855ec1f3232",
13-
"generated_at": "2026-02-19T05:51:35Z"
13+
"generated_at": "2026-02-19T20:01:55Z"
1414
}

examples/react-native/devbox.d/android/devices/devices.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@
1414
}
1515
],
1616
"checksum": "f5bfab3fdcbe8a23858954c18b1fa86d28a3316e801523aa6d4aa72ca9cf5ab7",
17-
"generated_at": "2026-02-19T19:12:22Z"
17+
"generated_at": "2026-02-19T20:57:49Z"
1818
}

examples/react-native/devbox.d/ios/devices/devices.lock

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@
1010
}
1111
],
1212
"checksum": "4d5276f203d7ad62860bfc067f76194df53be449d4aa8a3b2d069855ec1f3232",
13-
"generated_at": "2026-02-19T19:12:23Z"
13+
"generated_at": "2026-02-19T20:57:50Z"
1414
}

plugins/tests/ios/test-app-resolution.sh

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -158,14 +158,15 @@ assert_equal "$test_root/DerivedData/Build/Products/Real.app" "$result" \
158158
start_test "ios_find_app - fails when no .app found"
159159
test_root="$TMPDIR_BASE/test5"
160160
mkdir -p "$test_root"
161-
IOS_APP_ARTIFACT="" \
162-
assert_failure "ios_find_app '$test_root'" "Should fail when no .app exists"
161+
# Run from a clean dir so $PWD fallback search doesn't find fixture .app bundles
162+
(cd "$test_root" && IOS_APP_ARTIFACT="" \
163+
assert_failure "ios_find_app '$test_root'" "Should fail when no .app exists")
163164

164165
start_test "ios_find_app - error message includes guidance"
165166
test_root="$TMPDIR_BASE/test5b"
166167
mkdir -p "$test_root"
167-
IOS_APP_ARTIFACT="" \
168-
error_output=$(ios_find_app "$test_root" 2>&1 || true)
168+
# Run from a clean dir so $PWD fallback search doesn't find fixture .app bundles
169+
error_output=$(cd "$test_root" && IOS_APP_ARTIFACT="" ios_find_app "$test_root" 2>&1 || true)
169170
assert_contains "$error_output" "No .app bundle found" "Error should mention no .app found"
170171
assert_contains "$error_output" "IOS_APP_ARTIFACT" "Error should mention env var"
171172

plugins/tests/test-framework.sh

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ test_summary() {
235235
echo "Failed: $test_failed"
236236
echo ""
237237

238-
# Write results file for summary aggregation
238+
# Write per-suite results JSON
239239
local results_dir="${TEST_RESULTS_DIR:-$REPO_ROOT/reports/results}"
240240
if [[ ! "$results_dir" = /* ]]; then
241241
results_dir="$REPO_ROOT/$results_dir"
@@ -247,10 +247,14 @@ test_summary() {
247247
"suite": "${suite_name}",
248248
"passed": ${test_passed},
249249
"failed": ${test_failed},
250-
"total": ${total}
250+
"total": ${total},
251+
"timestamp": "$(date '+%Y-%m-%d %H:%M:%S')"
251252
}
252253
EOF
253254

255+
# Regenerate combined summary from all suite JSONs
256+
_regenerate_summary "$results_dir"
257+
254258
if [ "$test_failed" -gt 0 ]; then
255259
echo "RESULT: ✗ FAILED"
256260
exit 1
@@ -259,3 +263,86 @@ EOF
259263
exit 0
260264
fi
261265
}
266+
267+
# Regenerate reports/summary.md from all per-suite JSON files.
268+
# Called automatically by test_summary(); can also be called standalone.
269+
_regenerate_summary() {
270+
local results_dir="${1:-${TEST_RESULTS_DIR:-$REPO_ROOT/reports/results}}"
271+
local reports_dir="${REPORTS_DIR:-$REPO_ROOT/reports}"
272+
local logs_dir="${TEST_LOGS_DIR:-$reports_dir/logs}"
273+
274+
# Resolve relative paths against REPO_ROOT
275+
[[ "$results_dir" = /* ]] || results_dir="$REPO_ROOT/$results_dir"
276+
[[ "$reports_dir" = /* ]] || reports_dir="$REPO_ROOT/$reports_dir"
277+
[[ "$logs_dir" = /* ]] || logs_dir="$REPO_ROOT/$logs_dir"
278+
279+
local summary_file="$reports_dir/summary.md"
280+
281+
# Bail if no result files exist yet
282+
local json_files
283+
json_files=$(ls "$results_dir"/*.json 2>/dev/null | sort) || return 0
284+
[ -n "$json_files" ] || return 0
285+
286+
# Aggregate
287+
local _tp=0 _tf=0 _sc=0 _af=0
288+
local _names=() _passed=() _failed=() _totals=() _times=()
289+
290+
for rf in $json_files; do
291+
[ -f "$rf" ] || continue
292+
local n p f t ts
293+
n=$(jq -r '.suite // "unknown"' "$rf" 2>/dev/null)
294+
p=$(jq -r '.passed // 0' "$rf" 2>/dev/null)
295+
f=$(jq -r '.failed // 0' "$rf" 2>/dev/null)
296+
t=$(jq -r '.total // 0' "$rf" 2>/dev/null)
297+
ts=$(jq -r '.timestamp // ""' "$rf" 2>/dev/null)
298+
299+
_names+=("$n"); _passed+=("$p"); _failed+=("$f"); _totals+=("$t"); _times+=("$ts")
300+
_tp=$((_tp + p)); _tf=$((_tf + f)); _sc=$((_sc + 1))
301+
[ "$f" -gt 0 ] && _af=1
302+
done
303+
304+
local _gt=$((_tp + _tf))
305+
306+
mkdir -p "$reports_dir" 2>/dev/null || true
307+
{
308+
echo "# Test Suite Summary"
309+
echo ""
310+
echo "**Updated:** $(date '+%Y-%m-%d %H:%M:%S')"
311+
echo ""
312+
313+
if [ "$_af" -gt 0 ]; then
314+
echo "> **SOME TESTS FAILED**"
315+
else
316+
echo "> **ALL ${_sc} SUITES PASSED** (${_gt} tests)"
317+
fi
318+
echo ""
319+
320+
echo "| Suite | Passed | Failed | Total | Result | Ran At |"
321+
echo "|-------|-------:|-------:|------:|--------|--------|"
322+
323+
for i in $(seq 0 $((_sc - 1))); do
324+
local badge="PASS"
325+
[ "${_failed[$i]}" -gt 0 ] && badge="FAIL"
326+
echo "| ${_names[$i]} | ${_passed[$i]} | ${_failed[$i]} | ${_totals[$i]} | ${badge} | ${_times[$i]} |"
327+
done
328+
329+
local total_badge="PASS"
330+
[ "$_af" -gt 0 ] && total_badge="FAIL"
331+
echo "| **TOTAL** | **${_tp}** | **${_tf}** | **${_gt}** | **${total_badge}** | |"
332+
echo ""
333+
334+
# Log files
335+
if ls "$logs_dir"/*.txt >/dev/null 2>&1; then
336+
echo "## Log Files"
337+
echo ""
338+
for log in "$logs_dir"/*.txt; do
339+
echo "- \`$log\`"
340+
done
341+
echo ""
342+
fi
343+
344+
echo "---"
345+
echo ""
346+
echo "_Run \`devbox run test:fast\` to regenerate this summary_"
347+
} > "$summary_file"
348+
}

0 commit comments

Comments
 (0)