Skip to content

Commit 6e6dcf4

Browse files
david-allisondavid-allison
committed
test: add Roborazzi screenshot diff script
Allows a developer to compare from the current working tree to a known baseline using Roborazzi This produces `_compare.png` files if any screenshot tests detect changes. Usage: # compare the working tree tools/compare-screenshot-test.sh HEAD # compare to a past snapshot tools/compare-screenshot-test.sh 4d7daca The implementation uses * `git worktree` * `recordRoborazziPlayDebug` * `compareRoborazziPlayDebug` Somewhat related to issue 20942: screenshot tests in CI Assisted-by: Claude Opus 4.7 - everything except comments
1 parent c3def0a commit 6e6dcf4

4 files changed

Lines changed: 189 additions & 0 deletions

File tree

‎.idea/dictionaries/davidallison.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<component name="ProjectDictionaryState">
22
<dictionary name="davidallison">
33
<words>
4+
<w>.worktreeinclude</w>
45
<w>Aedict</w>
56
<w>Affero</w>
67
<w>Awaitable</w>

‎.idea/dictionaries/usernames.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
<w>Yadav</w>
8282
<w>Zaur</w>
8383
<w>bresan</w>
84+
<w>davidallison</w>
8485
<w>joshakaw</w>
8586
<w>krmanik</w>
8687
<w>lukstbit</w>

‎.worktreeinclude‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
# Includes the following files in Claude Code worktrees (gitignore syntax)
22
# https://code.claude.com/docs/en/worktrees#copy-gitignored-files-into-worktrees
3+
#
4+
# When modifying this file, consider if items should be copied in tools/compare-screenshot-test.sh
35
local.properties

‎tools/compare-screenshot-test.sh‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/bin/bash
2+
# Diff Roborazzi screenshot tests between a provided baseline commit and HEAD.
3+
#
4+
# Usage:
5+
# tools/compare-screenshot-test.sh <baseline-commit> [test-class-fqn]
6+
# tools/compare-screenshot-test.sh HEAD # diff working tree vs last commit
7+
#
8+
# With no test filter, runs every test tagged ScreenshotTestCategory.
9+
#
10+
# Implementation:
11+
# Baseline output is produced in a worktree: ../Anki-Android-screenshot-baseline
12+
13+
set -euo pipefail
14+
15+
# Ensure the commit hash is provided as the first argument
16+
if [ $# -lt 1 ]; then
17+
echo "usage: $(basename "$0") <baseline-commit> [test-class-fqn]" >&2
18+
echo " pass 'HEAD' to diff only your uncommitted changes" >&2
19+
exit 1
20+
fi
21+
BASELINE="$1"
22+
TEST_FILTER="${2:-}"
23+
24+
# Validate the baseline commit (arg 1)
25+
if ! git rev-parse --verify --quiet "${BASELINE}^{commit}" >/dev/null; then
26+
echo "error: baseline '$BASELINE' is not a valid git commit" >&2
27+
exit 1
28+
fi
29+
30+
BASELINE_SHA="$(git rev-parse --verify "$BASELINE")"
31+
HEAD_SHA="$(git rev-parse HEAD)"
32+
33+
# Fail if using HEAD with a clean working tree
34+
if [ "$BASELINE_SHA" = "$HEAD_SHA" ] \
35+
&& git diff --quiet HEAD && git diff --cached --quiet; then
36+
echo "error: nothing to compare. No unstaged changes." >&2
37+
exit 1
38+
fi
39+
40+
41+
# /Users/davidallison/StudioProjects/Anki-Android
42+
REPO_ROOT="$(git rev-parse --show-toplevel)"
43+
# /Users/davidallison/StudioProjects/Anki-Android-screenshot-baseline
44+
WORKTREE_DIR="${REPO_ROOT}/../Anki-Android-screenshot-baseline"
45+
OUT_DIR="${REPO_ROOT}/AnkiDroid/build/outputs/roborazzi"
46+
WORKTREE_OUT="${WORKTREE_DIR}/AnkiDroid/build/outputs/roborazzi"
47+
48+
# baseline 4d7daca42e test(card-browser): screenshot test
49+
# HEAD 18377ab3d1 feat(card-browser): enable edge to edge (+ uncommitted changes)
50+
echo "Comparing:"
51+
echo "baseline $(git log -1 --format='%h %s' "$BASELINE")"
52+
head_line="$(git log -1 --format='%h %s' HEAD)"
53+
if ! git diff --quiet HEAD || ! git diff --cached --quiet; then
54+
head_line="${head_line} (+ uncommitted changes)"
55+
fi
56+
echo "HEAD ${head_line}"
57+
# If the user picked a non-HEAD baseline, remind them HEAD is also valid
58+
# and is the quickest way to check just their uncommitted edits.
59+
if [ "$BASELINE_SHA" != "$HEAD_SHA" ]; then
60+
echo "Tip: pass 'HEAD' to diff only your uncommitted changes."
61+
fi
62+
echo
63+
64+
# Display a tip to use arg 2
65+
GRADLE_TESTS_ARG=()
66+
if [ -n "$TEST_FILTER" ]; then
67+
GRADLE_TESTS_ARG=(--tests "$TEST_FILTER")
68+
else
69+
echo "Tip: use the 2nd arg to limit test runs. Syntax: \"com.ichi2.anki.**Test\""
70+
echo
71+
fi
72+
73+
## Function definitions
74+
75+
# ANSI red, only when stdout is a TTY (no escape codes in pipes / CI logs).
76+
if [ -t 1 ]; then
77+
RED=$'\033[31m'
78+
NC=$'\033[0m'
79+
else
80+
RED='' NC=''
81+
fi
82+
83+
# Silence IPackageStatsObserver.java uses or overrides a deprecated API. Maintain progress output.
84+
gradle_quiet() {
85+
"$@" 2> >(grep -v -E '^Note:' >&2)
86+
}
87+
88+
# Open a directory in the OS file manager
89+
open_dir() {
90+
case "$(uname -s)" in
91+
Darwin) open "$1" ;;
92+
Linux) xdg-open "$1" >/dev/null 2>&1 ;;
93+
MINGW*|MSYS*|CYGWIN*) start "" "$1" ;;
94+
*) echo "warning: don't know how to open '$1' on $(uname -s)" >&2 ;;
95+
esac
96+
}
97+
98+
# Create or reuse the screenshot-baseline worktree (../Anki-Android-screenshot-baseline)
99+
worktree_head=""
100+
if [ -e "$WORKTREE_DIR/.git" ]; then
101+
worktree_head="$(git -C "$WORKTREE_DIR" rev-parse HEAD 2>/dev/null || true)"
102+
fi
103+
if [ "$worktree_head" = "$BASELINE_SHA" ]; then
104+
echo "==> Reusing existing worktree at $BASELINE"
105+
else
106+
git worktree remove --force "$WORKTREE_DIR" 2>/dev/null || true
107+
rm -rf "$WORKTREE_DIR" # handle a hard kill.
108+
git worktree add "$WORKTREE_DIR" "$BASELINE" >/dev/null
109+
fi
110+
111+
# Keep in sync with .worktreeinclude.
112+
# Fixes 'SDK location not found'
113+
if [ -f "${REPO_ROOT}/local.properties" ]; then
114+
cp "${REPO_ROOT}/local.properties" "${WORKTREE_DIR}/local.properties"
115+
fi
116+
117+
echo "==> Recording baseline at $BASELINE"
118+
gradle_quiet "${WORKTREE_DIR}/gradlew" -p "$WORKTREE_DIR" \
119+
:AnkiDroid:recordRoborazziPlayDebug -Pscreenshot -q \
120+
-x installGitHook \
121+
${GRADLE_TESTS_ARG[@]+"${GRADLE_TESTS_ARG[@]}"}
122+
123+
# Clear stale Roborazzi outputs
124+
gradle_quiet "${REPO_ROOT}/gradlew" -p "$REPO_ROOT" \
125+
:AnkiDroid:clearRoborazziPlayDebug -q
126+
127+
# Copy baselines to outputs/roborazzi/<TestClass>/**
128+
echo "==> Copying baselines to ${OUT_DIR#${REPO_ROOT}/}"
129+
staged=0
130+
shopt -s nullglob
131+
for class_dir in "${WORKTREE_OUT}"/*/; do
132+
[ -d "$class_dir" ] || continue
133+
class_name="$(basename "$class_dir")"
134+
pngs=("${class_dir}"*.png)
135+
if [ ${#pngs[@]} -eq 0 ]; then
136+
echo "warning: skipping empty class dir ${class_name}" >&2
137+
continue
138+
fi
139+
mkdir -p "${OUT_DIR}/${class_name}"
140+
cp "${pngs[@]}" "${OUT_DIR}/${class_name}/"
141+
staged=1
142+
done
143+
shopt -u nullglob
144+
145+
if [ $staged -eq 0 ]; then
146+
echo "error: baseline run produced no screenshots. Does the test filter '${TEST_FILTER:-(none)}' match any @Category(ScreenshotTestCategory) tests?" >&2
147+
exit 1
148+
fi
149+
150+
echo "==> Comparing baseline to HEAD"
151+
# *_actual.png / *_compare.png are written if there are differences
152+
gradle_quiet "${REPO_ROOT}/gradlew" -p "$REPO_ROOT" \
153+
:AnkiDroid:compareRoborazziPlayDebug -Pscreenshot -q \
154+
${GRADLE_TESTS_ARG[@]+"${GRADLE_TESTS_ARG[@]}"}
155+
156+
157+
shopt -s nullglob
158+
diffs=("${OUT_DIR}"/*_compare.png)
159+
shopt -u nullglob
160+
161+
if [ ${#diffs[@]} -eq 0 ]; then
162+
label="${TEST_FILTER:-all screenshot tests}"
163+
echo "No visual difference between $BASELINE and HEAD for ${label}."
164+
exit 0
165+
fi
166+
167+
# Screenshots were not equivalent. Explain and link the issues
168+
169+
# 1 visual diff(s):
170+
# diff: file:///Users/.../AnkiDroid/build/outputs/roborazzi/30_notes_compare.png
171+
echo "${RED}${#diffs[@]} visual diff(s)${NC} in file://${OUT_DIR}"
172+
for compare in "${diffs[@]}"; do
173+
echo " ${RED}diff:${NC} file://${compare}"
174+
done
175+
176+
# Offer to open AnkiDroid/build/outputs/roborazzi if running in a TTY
177+
if [ -t 0 ]; then
178+
read -n 1 -r -p "Press [R] to reveal files: "
179+
echo
180+
if [[ "$REPLY" =~ ^[Rr]$ ]]; then
181+
open_dir "$OUT_DIR"
182+
fi
183+
fi
184+
185+
exit 1

0 commit comments

Comments
 (0)