Skip to content

Commit 300a6ec

Browse files
committed
ci(native-pipeline): triage shard failures, capture screenshot diffs, bump iOS cap to 30m
1 parent 5e46bec commit 300a6ec

3 files changed

Lines changed: 188 additions & 26 deletions

File tree

.github/actions/archive-test-results/action.yaml

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,31 @@ runs:
3030
name: ${{ inputs.platform }}-screenshots-${{ inputs.test-type }}
3131
path: ${{ inputs.workspace-path }}/maestro/images/actual/${{ inputs.platform }}/**/*.png
3232
if-no-files-found: ignore
33-
33+
34+
- name: Collect screenshot diffs
35+
# On a failed assertScreenshot, Maestro writes the visual diff (the most useful artifact
36+
# for triaging a mismatch) next to the baseline — but under a *doubled* path, e.g.
37+
# maestro/images/expected/ios/maestro/images/expected/ios/<name>_diff.png, because it
38+
# resolves the diff name against the baseline's own relative path. That nested location
39+
# was never matched by the screenshots glob above, so diffs were silently lost. Gather
40+
# any *_diff.png from wherever it landed into one flat dir for upload.
41+
if: always()
42+
shell: bash
43+
run: |
44+
dest="${{ inputs.workspace-path }}/maestro/images/diff/${{ inputs.platform }}"
45+
mkdir -p "$dest"
46+
find "${{ inputs.workspace-path }}/maestro/images" -name '*_diff.png' \
47+
-not -path "$dest/*" -exec cp -f {} "$dest/" \; 2>/dev/null || true
48+
ls -1 "$dest" 2>/dev/null || true
49+
50+
- name: Archive screenshot diffs
51+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7
52+
if: always()
53+
with:
54+
name: ${{ inputs.platform }}-diffs-${{ inputs.test-type }}
55+
path: ${{ inputs.workspace-path }}/maestro/images/diff/${{ inputs.platform }}/*.png
56+
if-no-files-found: ignore
57+
3458
- name: Archive artifacts
3559
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f #v7
3660
if: always()
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Triage the widget/JS E2E shards of a finished run and print a Markdown report to stdout
4+
# (the aggregate-test-results job redirects it into $GITHUB_STEP_SUMMARY).
5+
#
6+
# The plain pass/fail table doesn't tell a reviewer *why* a shard is red — and most reds in
7+
# this pipeline are infra flakes (corrupt emulator image, runtime not up, missing artifact),
8+
# not real test failures. This script classifies each non-green shard from its job metadata
9+
# and, for the test step itself, from a grep over its log, so someone who didn't write this
10+
# branch can tell "re-run it" from "an actual widget broke" at a glance.
11+
#
12+
# Usage: triage-test-results.sh [<repo> [<run_id>]]
13+
# repo defaults to $GITHUB_REPOSITORY
14+
# run_id defaults to $GITHUB_RUN_ID
15+
# Requires: gh (authenticated via $GH_TOKEN), jq.
16+
17+
set -euo pipefail
18+
19+
REPO="${1:-${GITHUB_REPOSITORY:?repo required}}"
20+
RUN_ID="${2:-${GITHUB_RUN_ID:?run_id required}}"
21+
22+
# Pull every shard job (with its steps) once.
23+
jobs_json="$(gh api --paginate \
24+
"repos/${REPO}/actions/runs/${RUN_ID}/jobs?per_page=100" \
25+
--jq '[.jobs[] | select(.name | test("widget-tests|js-tests"))]' \
26+
| jq -s 'add // []')"
27+
28+
# Cache the per-job log lookups so we hit the API at most once per failing shard.
29+
fetch_log() {
30+
local job_id="$1" cache
31+
cache="$(mktemp)"
32+
gh api "repos/${REPO}/actions/jobs/${job_id}/logs" >"$cache" 2>/dev/null || true
33+
printf '%s' "$cache"
34+
}
35+
36+
# Sub-classify a failed "Run … tests" step from its log. Patterns are ordered most-specific
37+
# first; an emulator that never boots can't also mismatch a screenshot, so they're exclusive.
38+
classify_test_step() {
39+
local log="$1"
40+
if grep -qiE 'Error reading Zip content from a SeekableByteChannel|could not connect to TCP port 5554|Timeout waiting for emulator|emulator: ERROR' "$log"; then
41+
echo '📵 Android emulator/adb never booted — corrupt system image (infra flake, re-run)'
42+
elif grep -qE 'Smoke check FAILED' "$log"; then
43+
echo '🚬 App did not launch — smoke check failed (build/bundle broken)'
44+
elif grep -qiE 'threshold not met|Screenshot comparison failed|Comparison error: Assert screenshot' "$log"; then
45+
echo '🖼 Screenshot mismatch — visual regression or stale baseline (REAL failure)'
46+
elif grep -qiE 'driver.*(failed to start|did not start)|Unable to launch.*driver|Failed to connect to 127\.0\.0\.1' "$log"; then
47+
echo '🤖 Maestro driver did not attach (flake)'
48+
else
49+
echo '❓ Test step failed — cause unclear, open the job log'
50+
fi
51+
}
52+
53+
# Decide the triage cell for one job. Echoes "<cause>\t<bucket>" where bucket is
54+
# real | flake | none (none = green/skipped, not counted as a failure).
55+
triage_job() {
56+
local job="$1"
57+
local conclusion failed_step job_id log cause
58+
conclusion="$(jq -r '.conclusion // .status' <<<"$job")"
59+
job_id="$(jq -r '.id' <<<"$job")"
60+
61+
case "$conclusion" in
62+
success) printf '—\tnone'; return ;;
63+
skipped) printf '—\tnone'; return ;;
64+
cancelled)
65+
printf '⏱ Timed out — hit the per-shard timeout-minutes cap (slow/wedged shard)\tflake'
66+
return ;;
67+
esac
68+
69+
# failure (or anything unexpected): lead with the step that actually failed.
70+
failed_step="$(jq -r 'first(.steps[]? | select(.conclusion == "failure") | .name) // ""' <<<"$job")"
71+
case "$failed_step" in
72+
*Download*) printf '📦 Required artifact missing — upstream build did not publish it\tflake'; return ;;
73+
*"Start Mendix runtime"*) printf '🔌 Mendix runtime never became ready\tflake'; return ;;
74+
*"Run "*tests*)
75+
log="$(fetch_log "$job_id")"
76+
cause="$(classify_test_step "$log")"
77+
rm -f "$log"
78+
case "$cause" in
79+
*REAL*|*broken*) printf '%s\treal' "$cause" ;;
80+
*) printf '%s\tflake' "$cause" ;;
81+
esac
82+
return ;;
83+
"") printf '❓ Failed, but no single step is marked failed — open the job log\tflake'; return ;;
84+
*) printf '🧰 Setup/infra step failed: %s\tflake' "$failed_step"; return ;;
85+
esac
86+
}
87+
88+
result_badge() {
89+
case "$1" in
90+
success) echo '✅ pass' ;;
91+
failure) echo '❌ fail' ;;
92+
cancelled) echo '🚫 cancelled' ;;
93+
skipped) echo '⏭️ skipped' ;;
94+
*) echo "$1" ;;
95+
esac
96+
}
97+
98+
# --- Render -----------------------------------------------------------------------------
99+
# Sort so the rows that need a human come first: real failures, then infra/flake, then green.
100+
# Each emitted line is "<priority>\t<markdown row>"; we sort on priority then strip it.
101+
pass=0; real=0; flake=0; other=0
102+
rows=""
103+
while IFS= read -r job; do
104+
[ -z "$job" ] && continue
105+
name="$(jq -r '.name' <<<"$job")"
106+
conclusion="$(jq -r '.conclusion // .status' <<<"$job")"
107+
IFS=$'\t' read -r cause bucket <<<"$(triage_job "$job")"
108+
case "$conclusion" in
109+
success|skipped) pass=$((pass + 1)); prio=3 ;;
110+
*) case "$bucket" in
111+
real) real=$((real + 1)); prio=0 ;;
112+
flake) flake=$((flake + 1)); prio=1 ;;
113+
*) other=$((other + 1)); prio=2 ;;
114+
esac ;;
115+
esac
116+
rows+="${prio} | ${name} | $(result_badge "$conclusion") | ${cause} |"$'\n'
117+
done < <(jq -c '.[]' <<<"$jobs_json")
118+
119+
echo "## Native widget E2E results"
120+
echo ""
121+
echo "**${pass} green** · **${real} real failure(s)** 🖼🚬 · **${flake} likely infra/flake** (usually clears on re-run)"
122+
echo ""
123+
echo "| Shard | Result | Likely cause |"
124+
echo "| --- | --- | --- |"
125+
# Stable sort by priority (col 1), keeping discovery order within a bucket; drop the key column.
126+
printf '%s' "$rows" | sort -s -t$'\t' -k1,1n | cut -f2-
127+
echo ""
128+
echo "<sub>🖼 screenshot mismatch · 🚬 app/build broken · 📵 emulator · 🔌 runtime · 📦 artifact · 🤖 driver · ⏱ timeout. "
129+
echo "Only 🖼 and 🚬 point at the code under test; the rest are environment/flake and a re-run usually clears them.</sub>"

.github/workflows/NativePipeline.yml

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -744,10 +744,13 @@ jobs:
744744
# Run if widgets need testing (widgets_to_test is not empty) and project succeeds
745745
if: ${{ needs.scope.outputs.widgets_to_test != '[]' && always() && needs.project.result == 'success' && (needs.ios-app.result == 'success' || needs.ios-app.result == 'skipped') }}
746746
runs-on: macos-26
747-
# 20 min per widget shard (see android-widget-tests rationale). iOS needs the larger budget
748-
# more than Android: sim cold-boot + Maestro driver startup + runtime-ready wait eat several
749-
# minutes before any flow runs, and slow-but-healthy iOS shards were getting truncated at 15.
750-
timeout-minutes: 20
747+
# 30 min per widget shard, matching android-widget-tests. iOS needs the budget more than
748+
# Android: sim cold-boot + Maestro driver startup + runtime-ready wait eat several minutes
749+
# before any flow runs, so on a free macOS runner slow-but-healthy shards were getting
750+
# truncated — at 20 min several shards (accordion, background-image, bottom-sheet,
751+
# color-picker, safe-area-view) were cancelled mid-flow. If a shard still hits 30 the cause
752+
# is real app instability, not the cap.
753+
timeout-minutes: 30
751754
strategy:
752755
max-parallel: 4
753756
matrix:
@@ -954,32 +957,29 @@ jobs:
954957
# screenshots/logs we want to inspect.
955958
if: ${{ always() }}
956959
runs-on: ubuntu-26.04
960+
# The workflow-level block grants only packages:write, so every other scope defaults to
961+
# none. This job needs contents:read (to check out the triage script) and actions:read
962+
# (the script reads each shard's conclusion and logs via the Actions API).
963+
permissions:
964+
contents: read
965+
actions: read
957966
steps:
967+
- name: "Check out code"
968+
# Needed only for the triage script below; the merge steps don't touch the repo.
969+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
970+
with:
971+
fetch-depth: 1
958972
- name: "Write test result summary"
959-
# Consolidated pass/fail table for every widget/JS shard, rendered on the run's Summary
960-
# tab. Each shard's result lives in its own matrix job; we read them back from the API
961-
# (this job `needs:` them all, so conclusions are final by the time this runs).
973+
# Triage table for every widget/JS shard, rendered on the run's Summary tab. Beyond
974+
# pass/fail it classifies each red shard (timeout / emulator / runtime / artifact /
975+
# screenshot mismatch / …) so a reviewer can tell a real failure from an infra flake
976+
# without opening logs. Each shard's result lives in its own matrix job; the script
977+
# reads them back from the API (this job `needs:` them all, so conclusions are final).
962978
env:
963979
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
964980
run: |
965-
{
966-
echo "## Native widget E2E results"
967-
echo ""
968-
echo "| Shard | Result |"
969-
echo "| --- | --- |"
970-
} >> "$GITHUB_STEP_SUMMARY"
971-
gh api --paginate \
972-
"repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
973-
--jq '.jobs[]
974-
| select(.name | test("widget-tests|js-tests"))
975-
| "| \(.name) | " + (
976-
if .conclusion == "success" then "✅ pass"
977-
elif .conclusion == "failure" then "❌ fail"
978-
elif .conclusion == "cancelled" then "🚫 cancelled"
979-
elif .conclusion == "skipped" then "⏭️ skipped"
980-
else (.conclusion // .status)
981-
end) + " |"' \
982-
>> "$GITHUB_STEP_SUMMARY"
981+
bash .github/scripts/triage-test-results.sh \
982+
"${{ github.repository }}" "${{ github.run_id }}" >> "$GITHUB_STEP_SUMMARY"
983983
- name: "Merge screenshots"
984984
uses: actions/upload-artifact/merge@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
985985
# continue-on-error: a run that produced no screenshots (e.g. only JS-action tests) has
@@ -991,6 +991,15 @@ jobs:
991991
# Keep each shard's PNGs in its own subdir so identically-named actual/ files don't collide.
992992
separate-directories: true
993993
delete-merged: true
994+
- name: "Merge screenshot diffs"
995+
uses: actions/upload-artifact/merge@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
996+
# Only mismatching flows produce a diff; a run with no screenshot failures has none.
997+
continue-on-error: true
998+
with:
999+
name: all-screenshot-diffs
1000+
pattern: "*-diffs-*"
1001+
separate-directories: true
1002+
delete-merged: true
9941003
- name: "Merge runtime logs"
9951004
uses: actions/upload-artifact/merge@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
9961005
continue-on-error: true

0 commit comments

Comments
 (0)