CI: Add Roborazzi Screenshot comparison GitHub Action

.github/workflows/screenshot_comment.yml

@@ -0,0 +1,165 @@
+# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/CompareScreenshotComment.yml
+#
+# Copyright 2023 takahirom
+# Copyright 2019 Square, Inc.
+# Copyright The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "🛠️ Screenshots: Comment"
+
+on:
+  workflow_run:
+    workflows:
+      - "🛠️ Screenshots: Compare"
+    types:
+      - completed
+
+permissions: { }
+
+jobs:
+  Comment-CompareScreenshot:
+    name: comment
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+
+    timeout-minutes: 2
+
+    permissions:
+      actions: read # for downloading artifacts and reading the run id
+      pull-requests: write # for creating a comment on pull requests
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Download PR number artifact
+        uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
+        with:
+          name: pr
+          run_id: ${{ github.event.workflow_run.id }}
+      - id: get-pull-request-number
+        name: Get pull request number
+        shell: bash
+        run: |
+          echo "pull_request_number=$(cat NR)" >> "$GITHUB_OUTPUT"
+      - name: Download screenshot diff artifact
+        uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3.1.4
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          name: screenshot-diff
+          path: screenshot-diff
+      - id: check-if-there-are-valid-files
+        name: Check if there are valid files
+        shell: bash
+        run: |
+          # Find Roborazzi diff PNGs in the downloaded artifact (in <TestClass>/).
+          mapfile -t files_to_add < <(find . -name "*_compare.png" -type f)
+          if [ ${#files_to_add[@]} -gt 0 ]; then
+            exist_valid_files="true"
+          else
+            exist_valid_files="false"
+          fi
+          echo "exist_valid_files=$exist_valid_files" >> "$GITHUB_OUTPUT"
+      - id: generate-diff-reports
+        name: Generate diff reports
+        if: steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+          BASE_REF: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].base.ref || github.event.repository.default_branch }}
+        shell: bash
+        run: |
+          # Find Roborazzi diff PNGs to mention in the comment (in <TestClass>/).
+          mapfile -t files < <(find . -name "*_compare.png" -type f | sort)
+          total=${#files[@]}
+          # Cap the full list so we remain within GitHub's comment limit
+          max_details=100
+
+          # Group by test class - the file's immediate parent directory.
+          declare -A class_counts class_files
+          for file in "${files[@]}"; do
+            class_dir="${file%/*}"
+            class="${class_dir##*/}"
+            class_counts[$class]=$(( ${class_counts[$class]:-0} + 1 ))
+            class_files[$class]+="$(basename "$file")"$'\n'
+          done
+          mapfile -t classes < <(printf '%s\n' "${!class_counts[@]}" | sort)
+
+          # Resolve the artifact ID so we can deep-link to its download page.
+          artifact_id=$(gh api "repos/$REPO/actions/runs/$RUN_ID/artifacts" \
+            --jq '.artifacts[] | select(.name=="screenshot-diff") | .id')
+          artifact_url="${{ github.server_url }}/$REPO/actions/runs/$RUN_ID/artifacts/$artifact_id"
+
+          # delimiter for multi-line output to GITHUB_OUTPUT
+          delimiter="$(openssl rand -hex 8)"
+          {
+            echo "reports<<${delimiter}"
+            echo "Snapshot diff report vs \`$BASE_REF\`. Open [\`screenshot-diff\`]($artifact_url) for diffs."
+            echo ""
+            for class in "${classes[@]}"; do
+              count=${class_counts[$class]}
+              noun=$([ "$count" -eq 1 ] && echo change || echo changes)
+              echo "- **$class**: $count $noun"
+            done
+            echo ""
+            echo "<details><summary>All $total changed screenshots</summary>"
+            echo ""
+            remaining=$max_details
+            for class in "${classes[@]}"; do
+              (( remaining > 0 )) || break
+              echo "**$class**"
+              while IFS= read -r name; do
+                [ -z "$name" ] && continue
+                (( remaining > 0 )) || break
+                echo "- \`$name\`"
+                remaining=$(( remaining - 1 ))
+              done <<< "${class_files[$class]}"
+              echo ""
+            done
+            if (( total > max_details )); then
+              echo "_…and $((total - max_details)) more not shown._"
+            fi
+            echo "</details>"
+            echo "${delimiter}"
+          } >> "$GITHUB_OUTPUT"
+      # Run unconditionally so we can clear a stale comment when regressions are fixed.
+      - name: Find Comment
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: fc
+        with:
+          issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Snapshot diff report
+
+      - name: Add or update comment on PR
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        if: steps.generate-diff-reports.outputs.reports != ''
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ steps.get-pull-request-number.outputs.pull_request_number }}
+          body: ${{ steps.generate-diff-reports.outputs.reports }}
+          edit-mode: replace
+
+      # If a previous run posted a diff comment but this run has none, the regressions are fixed.
+      - name: Mark previous regressions as resolved
+        if: >
+          steps.check-if-there-are-valid-files.outputs.exist_valid_files == 'false' &&
+          steps.fc.outputs.comment-id != ''
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            Snapshot diff report: Previous regressions were resolved by the latest commit.
+          edit-mode: replace

.github/workflows/screenshot_compare.yml

@@ -0,0 +1,120 @@
+# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/CompareScreenshot.yml
+# Modified from the original: adapted to AnkiDroid CI conventions.
+#
+# Copyright 2023 takahirom
+# Copyright 2019 Square, Inc.
+# Copyright The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "🛠️ Screenshots: Compare"  # do not rename - referenced by screenshot_comment.yml
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions: {}
+
+jobs:
+  compare-screenshot-test:
+    name: compare
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    permissions:
+      contents: read # for clone
+      actions: write # for upload-artifact
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Configure JDK
+        uses: actions/setup-java@v5
+        with:
+          distribution: "temurin"
+          java-version: "21"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v6
+        with:
+          # Use open source provider: https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#basic-caching
+          cache-provider: basic
+          gradle-version: wrapper
+
+      - name: Download base branch screenshots
+        uses: dawidd6/action-download-artifact@v3
+        continue-on-error: true
+        with:
+          name: screenshot
+          workflow: screenshot_store.yml
+          branch: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}
+
+      - name: Compare screenshots
+        id: compare-screenshot-test
+        run: |
+          ./gradlew compareRoborazziPlayDebug -Pscreenshot --stacktrace
+
+      - name: Stage screenshot diffs
+        if: ${{ always() }}
+        env:
+          # Folder name inside the artifact (e.g. "screenshot-diff-pr-12345" on PRs, "screenshot-diff-main" on push)
+          SUBDIR: ${{ github.event_name == 'pull_request' && format('screenshot-diff-pr-{0}', github.event.number) || format('screenshot-diff-{0}', github.ref_name) }}
+        run: |
+          # Trim 'AnkiDroid/build/outputs/roborazzi' from the artifact
+          # Only keep diagnostic content for failing cases: each <class>/diffs/ becomes <class>/
+          mkdir -p "screenshot-diff/$SUBDIR"
+          shopt -s nullglob
+          for diffs_dir in AnkiDroid/build/outputs/roborazzi/*/diffs; do
+            class_name="$(basename "$(dirname "$diffs_dir")")"
+            mv "$diffs_dir" "screenshot-diff/$SUBDIR/$class_name"
+          done
+
+      - name: Upload screenshot diffs
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff  # referenced by screenshot_comment.yml
+          path: screenshot-diff
+          retention-days: 30
+
+      - name: Upload screenshot diff reports
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff-reports
+          path: |
+            **/build/reports
+          retention-days: 30
+
+      - name: Upload screenshot diff test results
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot-diff-test-results
+          path: |
+            **/build/test-results
+          retention-days: 30
+
+      - name: Save PR number
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          mkdir -p ./pr
+          echo ${{ github.event.number }} > ./pr/NR
+      - name: Persist PR number
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr  # downloaded by screenshot_comment.yml
+          path: pr/

.github/workflows/screenshot_store.yml

@@ -0,0 +1,101 @@
+# ⚠️ Do not rename the file - screenshot_store.yml is referenced by screenshot_compare.yml
+#
+# Adapted from https://github.com/takahirom/roborazzi/blob/4be7f304fa23f2f00fad67ab612aec2035ac9db2/.github/workflows/StoreScreenshot.yml
+# Modified from the original: adapted to AnkiDroid CI conventions.
+#
+# Copyright 2023 takahirom
+# Copyright 2019 Square, Inc.
+# Copyright The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+name: "🛠️ Screenshots: Store Baseline"
+
+on:
+  push:
+    branches:
+      - main
+  # On PRs, only run this when structural changes occur, to be sure 'main' can still store baselines
+  pull_request:
+    paths:
+      - '.github/workflows/screenshot_*.yml'
+      - 'gradle/libs.versions.toml'
+      - 'AnkiDroid/build.gradle'
+      - 'AnkiDroid/build.gradle.kts' # future-proof
+      - '**/ScreenshotTest.kt'
+      - 'tools/compare-screenshot-test.sh'
+
+permissions: {}
+
+jobs:
+  store-screenshot-test:
+    # On PRs, this verifies the workflow runs on `main`, it's not a store
+    name: ${{ github.ref == 'refs/heads/main' && 'store' || 'test' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    permissions:
+      contents: read # for clone
+      actions: write # for upload-artifact
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Configure JDK
+        uses: actions/setup-java@v5
+        with:
+          distribution: "temurin"
+          java-version: "21"
+
+      # Better than caching and/or extensions of actions/setup-java
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v6
+        with:
+          # Use open source provider: https://github.com/gradle/actions/blob/main/docs/setup-gradle.md#basic-caching
+          cache-provider: basic
+          gradle-version: wrapper
+
+      - name: Record screenshots
+        id: record-test
+        run: |
+          # Use --rerun-tasks to disable cache for test task
+          ./gradlew recordRoborazziPlayDebug -Pscreenshot --stacktrace --rerun-tasks
+
+      - name: Upload screenshot baseline
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot  # downloaded by screenshot_compare.yml
+          path: |
+            **/build/outputs/roborazzi
+          retention-days: 30
+
+      - name: Upload screenshot reports
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot-reports
+          path: |
+            **/build/reports
+          retention-days: 30
+
+      - name: Upload screenshot test results
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: screenshot-test-results
+          path: |
+            **/build/test-results
+          retention-days: 30