From 37bce38c997c4af07a3b57366c91c0315ba6875c Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison@users.noreply.github.com> Date: Tue, 5 May 2026 14:24:18 +0100 Subject: [PATCH 1/5] feat(ci): Screenshot diffing workflow Using Roborazzi for the implementation * 'screenshot_store': runs on `main` store an artifact with a baseline * 'screenshot_compare': an artifact is generated on PRs if changes are , detected against `main`'s artifact * 'screenshot_comment': if an artifact is generated, a PR comment is added Original sources https://github.com/takahirom/roborazzi/blob/main/.github/workflows/CompareScreenshot.yml https://github.com/takahirom/roborazzi/blob/main/.github/workflows/CompareScreenshotComment.yml https://github.com/takahirom/roborazzi/blob/main/.github/workflows/StoreScreenshot.yml - compare_screenshot_comment: quote the bot email so the [bot] brackets aren't interpreted by the shell - StudyScreenScreenshotTest: keep per-test-class subdirectory in the capture path so future screenshot tests don't collide - Use dedicated Roborazzi gradle tasks (compareRoborazziPlayDebug) - Remove pinned workflows to match repo style - Use open source gradle cache provider - Rename workflows - Add Apache header Issue 20942 Assisted-by: Claude Opus 4.7 --- .github/workflows/screenshot_comment.yml | 166 +++++++++++++++++++++++ .github/workflows/screenshot_compare.yml | 106 +++++++++++++++ .github/workflows/screenshot_store.yml | 92 +++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 .github/workflows/screenshot_comment.yml create mode 100644 .github/workflows/screenshot_compare.yml create mode 100644 .github/workflows/screenshot_store.yml diff --git a/.github/workflows/screenshot_comment.yml b/.github/workflows/screenshot_comment.yml new file mode 100644 index 000000000000..87994f9baa5f --- /dev/null +++ b/.github/workflows/screenshot_comment.yml @@ -0,0 +1,166 @@ +# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/CompareScreenshotComment.yml +# +# Copyright 2023 takahirom +# Copyright 2019 Square, Inc. +# Copyright The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "๐Ÿ› ๏ธ Screenshots: Comment" + +on: + workflow_run: + workflows: + - "๐Ÿ› ๏ธ Screenshots: Compare" + types: + - completed + +permissions: { } + +jobs: + Comment-CompareScreenshot: + name: comment + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + + timeout-minutes: 2 + + permissions: + actions: read # for downloading artifacts + contents: write # for pushing screenshot-diff to companion branch + pull-requests: write # for creating a comment on pull requests + + runs-on: ubuntu-latest + + steps: + - name: Download PR number artifact + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4 + with: + name: pr + run_id: ${{ github.event.workflow_run.id }} + - id: get-pull-request-number + name: Get pull request number + shell: bash + run: | + echo "pull_request_number=$(cat NR)" >> "$GITHUB_OUTPUT" + - name: main checkout + id: checkout-main + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }} + - id: switch-companion-branch + env: + BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} + run: | + # orphan means it will create no history branch + git branch -D "$BRANCH_NAME" || true + git checkout --orphan "$BRANCH_NAME" + git rm -rf . + - name: Download screenshot diff artifact + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4 + with: + run_id: ${{ github.event.workflow_run.id }} + name: screenshot-diff + path: screenshot-diff + - id: check-if-there-are-valid-files + name: Check if there are valid files + shell: bash + run: | + # Find all the files ending with _compare.png + mapfile -t files_to_add < <(find . -type f -name "*_compare.png") + + # Check for invalid file names and add only valid ones + exist_valid_files="false" + for file in "${files_to_add[@]}"; do + if [[ $file =~ ^[a-zA-Z0-9_./-]+$ ]]; then + exist_valid_files="true" + break + fi + done + echo "exist_valid_files=$exist_valid_files" >> "$GITHUB_OUTPUT" + - id: push-screenshot-diff + shell: bash + if: steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'true' + env: + BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} + run: | + # Find all the files ending with _compare.png + files_to_add=$(find . -type f -name "*_compare.png") + + # Check for invalid file names and add only valid ones + for file in $files_to_add; do + if [[ "$file" =~ ^[a-zA-Z0-9_./-]+$ ]]; then + git add "$file" + fi + done + git config --global user.name ScreenshotBot + git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m "Add screenshot diff" + git push origin HEAD:"$BRANCH_NAME" -f + - id: generate-diff-reports + name: Generate diff reports + if: steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'true' + env: + BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} + shell: bash + run: | + # Find all the files ending with _compare.png in roborazzi folder + files=$(find . -type f -name "*_compare.png" | grep "roborazzi/" | grep -E "^[a-zA-Z0-9_./-]+$") + delimiter="$(openssl rand -hex 8)" + { + echo "reports<<${delimiter}" + + # Create markdown table header + echo "Snapshot diff report vs base branch: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }}" + echo "| File name | Image |" + echo "|-------|-------|" + } >> "$GITHUB_OUTPUT" + + # Iterate over the files and create table rows + for file in $files; do + # Get the file name and insert newlines every 20 characters + fileName=$(basename "$file" | sed -r 's/(.{20})/\1
/g') + echo "| [$fileName](https://github.com/${{ github.repository }}/blob/$BRANCH_NAME/$file) | ![](https://github.com/${{ github.repository }}/blob/$BRANCH_NAME/$file?raw=true) |" >> "$GITHUB_OUTPUT" + done + echo "${delimiter}" >> "$GITHUB_OUTPUT" + - name: Find Comment + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 + id: fc + if: steps.generate-diff-reports.outputs.reports != '' + with: + issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }} + comment-author: 'github-actions[bot]' + body-includes: Snapshot diff report + + - name: Add or update comment on PR + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + if: steps.generate-diff-reports.outputs.reports != '' + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }} + body: ${{ steps.generate-diff-reports.outputs.reports }} + edit-mode: replace + + - name: Cleanup outdated companion branches + run: | + # Find outdated companion branches with last commit date + git branch -r --format="%(refname:lstrip=3)" | grep companion_ | while read -r branch; do + last_commit_date_timestamp=$(git log -1 --format=%ct "origin/$branch") + now_timestamp=$(date +%s) + # Delete branch if it's older than 1 month + if [ $((now_timestamp - last_commit_date_timestamp)) -gt 2592000 ]; then + echo "Deleting $branch" + git push origin --delete "$branch" + fi + done \ No newline at end of file diff --git a/.github/workflows/screenshot_compare.yml b/.github/workflows/screenshot_compare.yml new file mode 100644 index 000000000000..ffd141ba179b --- /dev/null +++ b/.github/workflows/screenshot_compare.yml @@ -0,0 +1,106 @@ +# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/CompareScreenshot.yml +# Modified from the original: adapted to AnkiDroid CI conventions. +# +# Copyright 2023 takahirom +# Copyright 2019 Square, Inc. +# Copyright The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "๐Ÿ› ๏ธ Screenshots: Compare" # do not rename - referenced by screenshot_comment.yml + +on: + push: + branches: + - main + pull_request: + +permissions: {} + +jobs: + compare-screenshot-test: + name: compare + runs-on: ubuntu-latest + timeout-minutes: 20 + + permissions: + contents: read # for clone + actions: write # for upload-artifact + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure JDK + uses: actions/setup-java@v5 + with: + distribution: "temurin" + java-version: "21" + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v6 + with: + # Use open source provider: https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#basic-caching + cache-provider: basic + gradle-version: wrapper + + - name: Download base branch screenshots + uses: dawidd6/action-download-artifact@v3 + continue-on-error: true + with: + name: screenshot + workflow: screenshot_store.yml + branch: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }} + + - name: Compare screenshots + id: compare-screenshot-test + run: | + ./gradlew compareRoborazziPlayDebug -Pscreenshot --stacktrace + + - name: Upload screenshot diffs + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot-diff # referenced by screenshot_comment.yml + path: | + **/build/outputs/roborazzi + retention-days: 30 + + - name: Upload screenshot diff reports + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot-diff-reports + path: | + **/build/reports + retention-days: 30 + + - name: Upload screenshot diff test results + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot-diff-test-results + path: | + **/build/test-results + retention-days: 30 + + - name: Save PR number + if: ${{ github.event_name == 'pull_request' }} + run: | + mkdir -p ./pr + echo ${{ github.event.number }} > ./pr/NR + - name: Persist PR number + uses: actions/upload-artifact@v4 + with: + name: pr # downloaded by screenshot_comment.yml + path: pr/ \ No newline at end of file diff --git a/.github/workflows/screenshot_store.yml b/.github/workflows/screenshot_store.yml new file mode 100644 index 000000000000..a433887b5f93 --- /dev/null +++ b/.github/workflows/screenshot_store.yml @@ -0,0 +1,92 @@ +# โš ๏ธ Do not rename the file - screenshot_store.yml is referenced by screenshot_compare.yml +# +# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/StoreScreenshot.yml +# Modified from the original: adapted to AnkiDroid CI conventions. +# +# Copyright 2023 takahirom +# Copyright 2019 Square, Inc. +# Copyright The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +name: "๐Ÿ› ๏ธ Screenshots: Store Baseline" + +on: + push: + branches: + - main + pull_request: + +permissions: {} + +jobs: + store-screenshot-test: + name: store + runs-on: ubuntu-latest + timeout-minutes: 20 + + permissions: + contents: read # for clone + actions: write # for upload-artifact + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Configure JDK + uses: actions/setup-java@v5 + with: + distribution: "temurin" + java-version: "21" + + # Better than caching and/or extensions of actions/setup-java + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v6 + with: + # Use open source provider: https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#basic-caching + cache-provider: basic + gradle-version: wrapper + + - name: Record screenshots + id: record-test + run: | + # Use --rerun-tasks to disable cache for test task + ./gradlew recordRoborazziPlayDebug -Pscreenshot --stacktrace --rerun-tasks + + - name: Upload screenshot baseline + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot # downloaded by screenshot_compare.yml + path: | + **/build/outputs/roborazzi + retention-days: 30 + + - name: Upload screenshot reports + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot-reports + path: | + **/build/reports + retention-days: 30 + + - name: Upload screenshot test results + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: screenshot-test-results + path: | + **/build/test-results + retention-days: 30 \ No newline at end of file From c4b9a3774b2a2de0c9c5184b4b3330a861556d16 Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison@users.noreply.github.com> Date: Mon, 4 May 2026 22:28:44 +0100 Subject: [PATCH 2/5] build(ci): use artifact for screenshot comparisons A branch was deemed too risky from a security perspective Assisted-by: Claude Opus 4.7 --- .github/workflows/screenshot_comment.yml | 139 +++++++++++------------ 1 file changed, 64 insertions(+), 75 deletions(-) diff --git a/.github/workflows/screenshot_comment.yml b/.github/workflows/screenshot_comment.yml index 87994f9baa5f..adfeb78b339c 100644 --- a/.github/workflows/screenshot_comment.yml +++ b/.github/workflows/screenshot_comment.yml @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: Handle the case where regressions are fixed + name: "๐Ÿ› ๏ธ Screenshots: Comment" on: @@ -37,8 +39,7 @@ jobs: timeout-minutes: 2 permissions: - actions: read # for downloading artifacts - contents: write # for pushing screenshot-diff to companion branch + actions: read # for downloading artifacts and reading the run id pull-requests: write # for creating a comment on pull requests runs-on: ubuntu-latest @@ -54,19 +55,6 @@ jobs: shell: bash run: | echo "pull_request_number=$(cat NR)" >> "$GITHUB_OUTPUT" - - name: main checkout - id: checkout-main - uses: actions/checkout@v4 - with: - ref: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }} - - id: switch-companion-branch - env: - BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} - run: | - # orphan means it will create no history branch - git branch -D "$BRANCH_NAME" || true - git checkout --orphan "$BRANCH_NAME" - git rm -rf . - name: Download screenshot diff artifact uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4 with: @@ -77,63 +65,77 @@ jobs: name: Check if there are valid files shell: bash run: | - # Find all the files ending with _compare.png - mapfile -t files_to_add < <(find . -type f -name "*_compare.png") - - # Check for invalid file names and add only valid ones - exist_valid_files="false" - for file in "${files_to_add[@]}"; do - if [[ $file =~ ^[a-zA-Z0-9_./-]+$ ]]; then - exist_valid_files="true" - break - fi - done + # Find roborazzi diff PNGs in the downloaded artifact. + mapfile -t files_to_add < <(find . -path '*/roborazzi/*' -name "*_compare.png" -type f) + if [ ${#files_to_add[@]} -gt 0 ]; then + exist_valid_files="true" + else + exist_valid_files="false" + fi echo "exist_valid_files=$exist_valid_files" >> "$GITHUB_OUTPUT" - - id: push-screenshot-diff - shell: bash - if: steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'true' - env: - BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} - run: | - # Find all the files ending with _compare.png - files_to_add=$(find . -type f -name "*_compare.png") - - # Check for invalid file names and add only valid ones - for file in $files_to_add; do - if [[ "$file" =~ ^[a-zA-Z0-9_./-]+$ ]]; then - git add "$file" - fi - done - git config --global user.name ScreenshotBot - git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' - git commit -m "Add screenshot diff" - git push origin HEAD:"$BRANCH_NAME" -f - id: generate-diff-reports name: Generate diff reports if: steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'true' env: - BRANCH_NAME: companion_${{ github.event.workflow_run.head_branch }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + BASE_REF: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }} shell: bash run: | - # Find all the files ending with _compare.png in roborazzi folder - files=$(find . -type f -name "*_compare.png" | grep "roborazzi/" | grep -E "^[a-zA-Z0-9_./-]+$") + # Find roborazzi diff PNGs to mention in the comment. + mapfile -t files < <(find . -path '*/roborazzi/*' -name "*_compare.png" -type f | sort) + total=${#files[@]} + # Cap the full list so we remain within GitHub's comment limit + max_details=100 + + # Group by test class โ€” the directory immediately under roborazzi/. + declare -A class_counts class_files + for file in "${files[@]}"; do + rest="${file#*/roborazzi/}" + class="${rest%%/*}" + class_counts[$class]=$(( ${class_counts[$class]:-0} + 1 )) + class_files[$class]+="$(basename "$file")"$'\n' + done + mapfile -t classes < <(printf '%s\n' "${!class_counts[@]}" | sort) + + # Resolve the artifact ID so we can deep-link to its download page. + artifact_id=$(gh api "repos/$REPO/actions/runs/$RUN_ID/artifacts" \ + --jq '.artifacts[] | select(.name=="screenshot-diff") | .id') + artifact_url="${{ github.server_url }}/$REPO/actions/runs/$RUN_ID/artifacts/$artifact_id" + + # delimiter for multi-line output to GITHUB_OUTPUT delimiter="$(openssl rand -hex 8)" { echo "reports<<${delimiter}" - - # Create markdown table header - echo "Snapshot diff report vs base branch: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }}" - echo "| File name | Image |" - echo "|-------|-------|" + echo "Snapshot diff report vs \`$BASE_REF\`. Open [\`screenshot-diff\`]($artifact_url) for diffs." + echo "" + for class in "${classes[@]}"; do + count=${class_counts[$class]} + noun=$([ "$count" -eq 1 ] && echo change || echo changes) + echo "- **$class**: $count $noun" + done + echo "" + echo "
All $total changed screenshots" + echo "" + remaining=$max_details + for class in "${classes[@]}"; do + (( remaining > 0 )) || break + echo "**$class**" + while IFS= read -r name; do + [ -z "$name" ] && continue + (( remaining > 0 )) || break + echo "- \`$name\`" + remaining=$(( remaining - 1 )) + done <<< "${class_files[$class]}" + echo "" + done + if (( total > max_details )); then + echo "_โ€ฆand $((total - max_details)) more not shown._" + fi + echo "
" + echo "${delimiter}" } >> "$GITHUB_OUTPUT" - - # Iterate over the files and create table rows - for file in $files; do - # Get the file name and insert newlines every 20 characters - fileName=$(basename "$file" | sed -r 's/(.{20})/\1
/g') - echo "| [$fileName](https://github.com/${{ github.repository }}/blob/$BRANCH_NAME/$file) | ![](https://github.com/${{ github.repository }}/blob/$BRANCH_NAME/$file?raw=true) |" >> "$GITHUB_OUTPUT" - done - echo "${delimiter}" >> "$GITHUB_OUTPUT" - name: Find Comment uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 id: fc @@ -150,17 +152,4 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }} body: ${{ steps.generate-diff-reports.outputs.reports }} - edit-mode: replace - - - name: Cleanup outdated companion branches - run: | - # Find outdated companion branches with last commit date - git branch -r --format="%(refname:lstrip=3)" | grep companion_ | while read -r branch; do - last_commit_date_timestamp=$(git log -1 --format=%ct "origin/$branch") - now_timestamp=$(date +%s) - # Delete branch if it's older than 1 month - if [ $((now_timestamp - last_commit_date_timestamp)) -gt 2592000 ]; then - echo "Deleting $branch" - git push origin --delete "$branch" - fi - done \ No newline at end of file + edit-mode: replace \ No newline at end of file From a08a2e27735559f3b3291164101e1b4d31e49ef4 Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison@users.noreply.github.com> Date: Tue, 5 May 2026 12:57:09 +0100 Subject: [PATCH 3/5] build(ci): improve 'screenshot-diff' directory structure * remove the nested folder tree (build/outputs/roborazzi) * name the directory: screenshot-diff.zip contains `screenshot-diff-pr-123` so conflicts don't occur and cleanup is easier Assisted-by: Claude Opus 4.7 - all Issue 20942 --- .github/workflows/screenshot_comment.yml | 14 +++++++------- .github/workflows/screenshot_compare.yml | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/screenshot_comment.yml b/.github/workflows/screenshot_comment.yml index adfeb78b339c..2405b7f27f35 100644 --- a/.github/workflows/screenshot_comment.yml +++ b/.github/workflows/screenshot_comment.yml @@ -65,8 +65,8 @@ jobs: name: Check if there are valid files shell: bash run: | - # Find roborazzi diff PNGs in the downloaded artifact. - mapfile -t files_to_add < <(find . -path '*/roborazzi/*' -name "*_compare.png" -type f) + # Find Roborazzi diff PNGs in the downloaded artifact (in /). + mapfile -t files_to_add < <(find . -name "*_compare.png" -type f) if [ ${#files_to_add[@]} -gt 0 ]; then exist_valid_files="true" else @@ -83,17 +83,17 @@ jobs: BASE_REF: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }} shell: bash run: | - # Find roborazzi diff PNGs to mention in the comment. - mapfile -t files < <(find . -path '*/roborazzi/*' -name "*_compare.png" -type f | sort) + # Find Roborazzi diff PNGs to mention in the comment (in /). + mapfile -t files < <(find . -name "*_compare.png" -type f | sort) total=${#files[@]} # Cap the full list so we remain within GitHub's comment limit max_details=100 - # Group by test class โ€” the directory immediately under roborazzi/. + # Group by test class - the file's immediate parent directory. declare -A class_counts class_files for file in "${files[@]}"; do - rest="${file#*/roborazzi/}" - class="${rest%%/*}" + class_dir="${file%/*}" + class="${class_dir##*/}" class_counts[$class]=$(( ${class_counts[$class]:-0} + 1 )) class_files[$class]+="$(basename "$file")"$'\n' done diff --git a/.github/workflows/screenshot_compare.yml b/.github/workflows/screenshot_compare.yml index ffd141ba179b..95ae924b98b8 100644 --- a/.github/workflows/screenshot_compare.yml +++ b/.github/workflows/screenshot_compare.yml @@ -67,13 +67,27 @@ jobs: run: | ./gradlew compareRoborazziPlayDebug -Pscreenshot --stacktrace + - name: Stage screenshot diffs + if: ${{ always() }} + env: + # Folder name inside the artifact (e.g. "screenshot-diff-pr-12345" on PRs, "screenshot-diff-main" on push) + SUBDIR: ${{ github.event_name == 'pull_request' && format('screenshot-diff-pr-{0}', github.event.number) || format('screenshot-diff-{0}', github.ref_name) }} + run: | + # Trim 'AnkiDroid/build/outputs/roborazzi' from the artifact + # Only keep diagnostic content for failing cases: each /diffs/ becomes / + mkdir -p "screenshot-diff/$SUBDIR" + shopt -s nullglob + for diffs_dir in AnkiDroid/build/outputs/roborazzi/*/diffs; do + class_name="$(basename "$(dirname "$diffs_dir")")" + mv "$diffs_dir" "screenshot-diff/$SUBDIR/$class_name" + done + - name: Upload screenshot diffs uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: screenshot-diff # referenced by screenshot_comment.yml - path: | - **/build/outputs/roborazzi + path: screenshot-diff retention-days: 30 - name: Upload screenshot diff reports From 19a148f4af2675caac3072257d55e2cab4cea179 Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison@users.noreply.github.com> Date: Tue, 5 May 2026 13:22:00 +0100 Subject: [PATCH 4/5] build(ci): update screenshot comment if resolved If the regressions were fixed, mark the PR as resolved Do not post a comment if one already existed Issue 20942 Assisted-by: Claude Opus 4.7 - all --- .github/workflows/screenshot_comment.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/screenshot_comment.yml b/.github/workflows/screenshot_comment.yml index 2405b7f27f35..7b2e99f9c24d 100644 --- a/.github/workflows/screenshot_comment.yml +++ b/.github/workflows/screenshot_comment.yml @@ -16,8 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Handle the case where regressions are fixed - name: "๐Ÿ› ๏ธ Screenshots: Comment" on: @@ -136,10 +134,10 @@ jobs: echo "" echo "${delimiter}" } >> "$GITHUB_OUTPUT" + # Run unconditionally so we can clear a stale comment when regressions are fixed. - name: Find Comment uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0 id: fc - if: steps.generate-diff-reports.outputs.reports != '' with: issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }} comment-author: 'github-actions[bot]' @@ -152,4 +150,16 @@ jobs: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }} body: ${{ steps.generate-diff-reports.outputs.reports }} + edit-mode: replace + + # If a previous run posted a diff comment but this run has none, the regressions are fixed. + - name: Mark previous regressions as resolved + if: > + steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'false' && + steps.fc.outputs.comment-id != '' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + body: | + Snapshot diff report: Previous regressions were resolved by the latest commit. edit-mode: replace \ No newline at end of file From 811c3800e0523178d2e4b38d44949d210ccd87fa Mon Sep 17 00:00:00 2001 From: David Allison <62114487+david-allison@users.noreply.github.com> Date: Tue, 5 May 2026 14:23:58 +0100 Subject: [PATCH 5/5] build(ci): don't store screenshots on most PRs Store only needs to be executed on `main` We run on some file changes, in case they break the upload process, but this is only defensive. Note: this does upload artifacts, this could be avoided, but the complexity makes the 'happy path' of this workflow harder to reason about. Issue 20942 Assisted-by: Claude Opus 4.7 --- .github/workflows/screenshot_store.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/screenshot_store.yml b/.github/workflows/screenshot_store.yml index a433887b5f93..067373499476 100644 --- a/.github/workflows/screenshot_store.yml +++ b/.github/workflows/screenshot_store.yml @@ -26,13 +26,22 @@ on: push: branches: - main + # On PRs, only run this when structural changes occur, to be sure 'main' can still store baselines pull_request: + paths: + - '.github/workflows/screenshot_*.yml' + - 'gradle/libs.versions.toml' + - 'AnkiDroid/build.gradle' + - 'AnkiDroid/build.gradle.kts' # future-proof + - '**/ScreenshotTest.kt' + - 'tools/compare-screenshot-test.sh' permissions: {} jobs: store-screenshot-test: - name: store + # On PRs, this verifies the workflow runs on `main`, it's not a store + name: ${{ github.ref == 'refs/heads/main' && 'store' || 'test' }} runs-on: ubuntu-latest timeout-minutes: 20