|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Triage the widget/JS E2E shards of a finished run and print a Markdown report to stdout |
| 4 | +# (the aggregate-test-results job redirects it into $GITHUB_STEP_SUMMARY). |
| 5 | +# |
| 6 | +# The plain pass/fail table doesn't tell a reviewer *why* a shard is red — and most reds in |
| 7 | +# this pipeline are infra flakes (corrupt emulator image, runtime not up, missing artifact), |
| 8 | +# not real test failures. This script classifies each non-green shard from its job metadata |
| 9 | +# and, for the test step itself, from a grep over its log, so someone who didn't write this |
| 10 | +# branch can tell "re-run it" from "an actual widget broke" at a glance. |
| 11 | +# |
| 12 | +# Usage: triage-test-results.sh [<repo> [<run_id>]] |
| 13 | +# repo defaults to $GITHUB_REPOSITORY |
| 14 | +# run_id defaults to $GITHUB_RUN_ID |
| 15 | +# Requires: gh (authenticated via $GH_TOKEN), jq. |
| 16 | + |
| 17 | +set -euo pipefail |
| 18 | + |
| 19 | +REPO="${1:-${GITHUB_REPOSITORY:?repo required}}" |
| 20 | +RUN_ID="${2:-${GITHUB_RUN_ID:?run_id required}}" |
| 21 | + |
| 22 | +# Pull every shard job (with its steps) once. |
| 23 | +jobs_json="$(gh api --paginate \ |
| 24 | + "repos/${REPO}/actions/runs/${RUN_ID}/jobs?per_page=100" \ |
| 25 | + --jq '[.jobs[] | select(.name | test("widget-tests|js-tests"))]' \ |
| 26 | + | jq -s 'add // []')" |
| 27 | + |
| 28 | +# Cache the per-job log lookups so we hit the API at most once per failing shard. |
| 29 | +fetch_log() { |
| 30 | + local job_id="$1" cache |
| 31 | + cache="$(mktemp)" |
| 32 | + gh api "repos/${REPO}/actions/jobs/${job_id}/logs" >"$cache" 2>/dev/null || true |
| 33 | + printf '%s' "$cache" |
| 34 | +} |
| 35 | + |
| 36 | +# Sub-classify a failed "Run … tests" step from its log. Patterns are ordered most-specific |
| 37 | +# first; an emulator that never boots can't also mismatch a screenshot, so they're exclusive. |
| 38 | +classify_test_step() { |
| 39 | + local log="$1" |
| 40 | + if grep -qiE 'Error reading Zip content from a SeekableByteChannel|could not connect to TCP port 5554|Timeout waiting for emulator|emulator: ERROR' "$log"; then |
| 41 | + echo '📵 Android emulator/adb never booted — corrupt system image (infra flake, re-run)' |
| 42 | + elif grep -qE 'Smoke check FAILED' "$log"; then |
| 43 | + echo '🚬 App did not launch — smoke check failed (build/bundle broken)' |
| 44 | + elif grep -qiE 'threshold not met|Screenshot comparison failed|Comparison error: Assert screenshot' "$log"; then |
| 45 | + echo '🖼 Screenshot mismatch — visual regression or stale baseline (REAL failure)' |
| 46 | + elif grep -qiE 'driver.*(failed to start|did not start)|Unable to launch.*driver|Failed to connect to 127\.0\.0\.1' "$log"; then |
| 47 | + echo '🤖 Maestro driver did not attach (flake)' |
| 48 | + else |
| 49 | + echo '❓ Test step failed — cause unclear, open the job log' |
| 50 | + fi |
| 51 | +} |
| 52 | + |
| 53 | +# Decide the triage cell for one job. Echoes "<cause>\t<bucket>" where bucket is |
| 54 | +# real | flake | none (none = green/skipped, not counted as a failure). |
| 55 | +triage_job() { |
| 56 | + local job="$1" |
| 57 | + local conclusion failed_step job_id log cause |
| 58 | + conclusion="$(jq -r '.conclusion // .status' <<<"$job")" |
| 59 | + job_id="$(jq -r '.id' <<<"$job")" |
| 60 | + |
| 61 | + case "$conclusion" in |
| 62 | + success) printf '—\tnone'; return ;; |
| 63 | + skipped) printf '—\tnone'; return ;; |
| 64 | + cancelled) |
| 65 | + printf '⏱ Timed out — hit the per-shard timeout-minutes cap (slow/wedged shard)\tflake' |
| 66 | + return ;; |
| 67 | + esac |
| 68 | + |
| 69 | + # failure (or anything unexpected): lead with the step that actually failed. |
| 70 | + failed_step="$(jq -r 'first(.steps[]? | select(.conclusion == "failure") | .name) // ""' <<<"$job")" |
| 71 | + case "$failed_step" in |
| 72 | + *Download*) printf '📦 Required artifact missing — upstream build did not publish it\tflake'; return ;; |
| 73 | + *"Start Mendix runtime"*) printf '🔌 Mendix runtime never became ready\tflake'; return ;; |
| 74 | + *"Run "*tests*) |
| 75 | + log="$(fetch_log "$job_id")" |
| 76 | + cause="$(classify_test_step "$log")" |
| 77 | + rm -f "$log" |
| 78 | + case "$cause" in |
| 79 | + *REAL*|*broken*) printf '%s\treal' "$cause" ;; |
| 80 | + *) printf '%s\tflake' "$cause" ;; |
| 81 | + esac |
| 82 | + return ;; |
| 83 | + "") printf '❓ Failed, but no single step is marked failed — open the job log\tflake'; return ;; |
| 84 | + *) printf '🧰 Setup/infra step failed: %s\tflake' "$failed_step"; return ;; |
| 85 | + esac |
| 86 | +} |
| 87 | + |
| 88 | +result_badge() { |
| 89 | + case "$1" in |
| 90 | + success) echo '✅ pass' ;; |
| 91 | + failure) echo '❌ fail' ;; |
| 92 | + cancelled) echo '🚫 cancelled' ;; |
| 93 | + skipped) echo '⏭️ skipped' ;; |
| 94 | + *) echo "$1" ;; |
| 95 | + esac |
| 96 | +} |
| 97 | + |
| 98 | +# --- Render ----------------------------------------------------------------------------- |
| 99 | +# Sort so the rows that need a human come first: real failures, then infra/flake, then green. |
| 100 | +# Each emitted line is "<priority>\t<markdown row>"; we sort on priority then strip it. |
| 101 | +pass=0; real=0; flake=0; other=0 |
| 102 | +rows="" |
| 103 | +while IFS= read -r job; do |
| 104 | + [ -z "$job" ] && continue |
| 105 | + name="$(jq -r '.name' <<<"$job")" |
| 106 | + conclusion="$(jq -r '.conclusion // .status' <<<"$job")" |
| 107 | + IFS=$'\t' read -r cause bucket <<<"$(triage_job "$job")" |
| 108 | + case "$conclusion" in |
| 109 | + success|skipped) pass=$((pass + 1)); prio=3 ;; |
| 110 | + *) case "$bucket" in |
| 111 | + real) real=$((real + 1)); prio=0 ;; |
| 112 | + flake) flake=$((flake + 1)); prio=1 ;; |
| 113 | + *) other=$((other + 1)); prio=2 ;; |
| 114 | + esac ;; |
| 115 | + esac |
| 116 | + rows+="${prio} | ${name} | $(result_badge "$conclusion") | ${cause} |"$'\n' |
| 117 | +done < <(jq -c '.[]' <<<"$jobs_json") |
| 118 | + |
| 119 | +echo "## Native widget E2E results" |
| 120 | +echo "" |
| 121 | +echo "**${pass} green** · **${real} real failure(s)** 🖼🚬 · **${flake} likely infra/flake** (usually clears on re-run)" |
| 122 | +echo "" |
| 123 | +echo "| Shard | Result | Likely cause |" |
| 124 | +echo "| --- | --- | --- |" |
| 125 | +# Stable sort by priority (col 1), keeping discovery order within a bucket; drop the key column. |
| 126 | +printf '%s' "$rows" | sort -s -t$'\t' -k1,1n | cut -f2- |
| 127 | +echo "" |
| 128 | +echo "<sub>🖼 screenshot mismatch · 🚬 app/build broken · 📵 emulator · 🔌 runtime · 📦 artifact · 🤖 driver · ⏱ timeout. " |
| 129 | +echo "Only 🖼 and 🚬 point at the code under test; the rest are environment/flake and a re-run usually clears them.</sub>" |
0 commit comments