test-vr.yml

# GitHub Actions visual regression testing workflow.
#
# Runs a Diffy visual regression comparison after a deployment lands.
# Triggered automatically by 'notify-diffy' (via repository_dispatch)
# when the deployed PR has the configured label, or manually via
# 'workflow_dispatch' against any URL.
name: Test - Visual regression

on:
  repository_dispatch:
    types:
      - vr_run
  workflow_dispatch:
    inputs:
      source_env:
        type: string
        description: 'Diffy source environment (production, staging, development)'
        required: false
        default: 'production'
      target_url:
        type: string
        description: 'Target URL to compare against'
        required: true
      label:
        type: string
        description: 'Diff label shown in Diffy UI'
        required: false
        default: 'manual'

permissions:
  contents: read
  pull-requests: read

defaults:
  run:
    shell: bash

concurrency:
  group: vr-${{ github.event.client_payload.target_url || github.event.inputs.target_url || github.run_id }}
  cancel-in-progress: true

env:
  DIFFY_CLI_VERSION: ${{ vars.VR_DIFFY_CLI_VERSION || '0.1.53' }}
  DIFFY_MAX_WAIT: ${{ vars.VR_DIFFY_MAX_WAIT || '2700' }}
  DIFFY_PR_LABEL: ${{ vars.VR_DIFFY_PR_LABEL || 'VR' }}
  DIFFY_AUTO_BRANCHES: ${{ vars.VR_DIFFY_AUTO_BRANCHES || 'deps/*' }}
  DIFFY_POLL_INTERVAL: ${{ vars.VR_DIFFY_POLL_INTERVAL || '30' }}
  SOURCE_ENV: ${{ github.event.client_payload.source_env || inputs.source_env || 'production' }}
  TARGET_URL: ${{ github.event.client_payload.target_url || inputs.target_url }}
  LABEL: ${{ github.event.client_payload.label || inputs.label || 'manual' }}

jobs:

  vr-compare:
    runs-on: ubuntu-latest
    permissions:
      contents: read # Read repository contents.
      pull-requests: read # 'gh pr view' reads PR labels and head branch.
    outputs:
      diff_id: ${{ steps.compare.outputs.diff_id }}
      pr_number: ${{ steps.resolve_pr.outputs.pr_number }}
      skipped: ${{ steps.gate.outputs.skipped }}
      changes_percent: ${{ steps.result.outputs.changes_percent }}
      shared_url: ${{ steps.result.outputs.shared_url }}
      pages_changed: ${{ steps.result.outputs.pages_changed }}
      pages_total: ${{ steps.result.outputs.pages_total }}

    steps:
      - name: Resolve PR number from target URL
        id: resolve_pr
        run: |
          set -euo pipefail

          # Hosting providers expose PR environments via URLs that contain
          # a 'pr-<number>' segment (e.g. 'app.pr-123.example.lagoon.cloud').
          # Extract the number from the URL; if no match, this is not a PR
          # deployment and visual regression should not run.
          pr_number="$(printf '%s' "${TARGET_URL}" | sed -n 's|.*pr-\([0-9]\{1,\}\).*|\1|p')"

          if [ -n "${pr_number}" ]; then
            echo "PR #${pr_number} resolved from target URL ${TARGET_URL}."
          else
            echo "::notice::No PR pattern found in target URL ${TARGET_URL}. Visual regression will not run."
          fi
          echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT"

      - name: Gate run on PR label (or skip for non-PR contexts on dispatch)
        id: gate
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ steps.resolve_pr.outputs.pr_number }}
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REPOSITORY: ${{ github.repository }}
        run: |
          set -euo pipefail

          # Manual workflow_dispatch always proceeds - the operator
          # explicitly asked for a comparison.
          if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then
            echo "Manual dispatch, proceeding."
            echo "skipped=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # repository_dispatch with no PR resolved - nothing to gate on.
          if [ -z "${PR_NUMBER}" ]; then
            echo "::notice::No PR associated with this deployment, skipping visual regression."
            echo "skipped=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # PR resolved - fetch labels and head branch in one call.
          pr_data="$(gh pr view "${PR_NUMBER}" --repo "${GITHUB_REPOSITORY}" --json labels,headRefName)"
          head_ref="$(printf '%s' "${pr_data}" | jq -r '.headRefName')"
          labels_lower="$(printf '%s' "${pr_data}" | jq -r '.labels[].name' | tr '[:upper:]' '[:lower:]')"

          # Auto-run bypass: PRs from configured branches (e.g. Renovate's
          # `deps/*`) skip the label check.
          if [ -n "${DIFFY_AUTO_BRANCHES}" ]; then
            IFS=',' read -ra patterns <<<"${DIFFY_AUTO_BRANCHES}"
            for pattern in "${patterns[@]}"; do
              # Trim surrounding whitespace.
              pattern="${pattern# }"
              pattern="${pattern% }"
              [ -z "${pattern}" ] && continue
              # shellcheck disable=SC2254
              case "${head_ref}" in
                ${pattern})
                  echo "PR #${PR_NUMBER} head branch '${head_ref}' matches auto-run pattern '${pattern}', proceeding."
                  echo "skipped=false" >> "$GITHUB_OUTPUT"
                  exit 0
                  ;;
              esac
            done
          fi

          # Label check (case-insensitive).
          needle="$(printf '%s' "${DIFFY_PR_LABEL}" | tr '[:upper:]' '[:lower:]')"
          if echo "${labels_lower}" | grep -qx "${needle}"; then
            echo "PR #${PR_NUMBER} has the '${DIFFY_PR_LABEL}' label, proceeding."
            echo "skipped=false" >> "$GITHUB_OUTPUT"
          else
            echo "::notice::PR #${PR_NUMBER} does not have the '${DIFFY_PR_LABEL}' label and head branch '${head_ref}' does not match VR_DIFFY_AUTO_BRANCHES, skipping visual regression."
            echo "skipped=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Validate target URL
        if: steps.gate.outputs.skipped != 'true'
        run: |
          set -euo pipefail
          if [ -z "${TARGET_URL}" ]; then
            echo "::error::TARGET_URL is empty."
            exit 1
          fi
          echo "Target URL: ${TARGET_URL}"

      - name: Install Diffy CLI
        if: steps.gate.outputs.skipped != 'true'
        run: |
          set -euo pipefail
          curl -fsSL -o /tmp/diffy.phar "https://github.com/DiffyWebsite/diffy-cli/releases/download/${DIFFY_CLI_VERSION}/diffy.phar"
          chmod +x /tmp/diffy.phar
          /tmp/diffy.phar --version

      - name: Authenticate Diffy CLI
        if: steps.gate.outputs.skipped != 'true'
        env:
          DIFFY_API_KEY: ${{ secrets.VR_DIFFY_API_KEY }}
        run: |
          set -euo pipefail
          /tmp/diffy.phar auth:login "${DIFFY_API_KEY}"

      - name: Start Diffy comparison
        id: compare
        if: steps.gate.outputs.skipped != 'true'
        env:
          DIFFY_PROJECT_ID: ${{ vars.VR_DIFFY_PROJECT_ID }}
          VR_SOURCE_HTTP_USER: ${{ secrets.VR_SOURCE_HTTP_USER }}
          VR_SOURCE_HTTP_PASS: ${{ secrets.VR_SOURCE_HTTP_PASS }}
          VR_TARGET_HTTP_USER: ${{ secrets.VR_TARGET_HTTP_USER }}
          VR_TARGET_HTTP_PASS: ${{ secrets.VR_TARGET_HTTP_PASS }}
        run: |
          set -euo pipefail
          echo "Starting Diffy comparison: source=${SOURCE_ENV} target=${TARGET_URL} label=${LABEL}"

          # HTTP basic auth credentials are passed conditionally. The Diffy CLI
          # forwards user/pass options as-is to the API, and the SDK gates on
          # PHP isset() - an empty string still triggers basic-auth mode with
          # empty credentials, which then fails against the upstream server.
          # Only pass the flags when both user and pass are non-empty.
          auth_args=()
          if [ -n "${VR_SOURCE_HTTP_USER}" ] && [ -n "${VR_SOURCE_HTTP_PASS}" ]; then
            auth_args+=(--env1User="${VR_SOURCE_HTTP_USER}" --env1Pass="${VR_SOURCE_HTTP_PASS}")
          fi
          if [ -n "${VR_TARGET_HTTP_USER}" ] && [ -n "${VR_TARGET_HTTP_PASS}" ]; then
            auth_args+=(--env2User="${VR_TARGET_HTTP_USER}" --env2Pass="${VR_TARGET_HTTP_PASS}")
          fi

          diff_id="$(/tmp/diffy.phar project:compare \
            "${DIFFY_PROJECT_ID}" \
            "${SOURCE_ENV}" \
            custom \
            --env2Url="${TARGET_URL}" \
            --name="${LABEL}" \
            ${auth_args[@]+"${auth_args[@]}"})"

          if [ -z "${diff_id}" ]; then
            echo "::error::Diffy did not return a diff ID."
            exit 1
          fi

          echo "Diffy diff ID: ${diff_id}"
          echo "diff_id=${diff_id}" >> "$GITHUB_OUTPUT"

      - name: Poll Diffy until comparison completes
        if: steps.gate.outputs.skipped != 'true'
        run: |
          set -euo pipefail
          diff_id="${STEPS_COMPARE_OUTPUTS_DIFF_ID}"
          start=$(date +%s)
          attempt=0

          while :; do
            attempt=$((attempt + 1))
            elapsed=$(($(date +%s) - start))

            if [ "${elapsed}" -ge "${DIFFY_MAX_WAIT}" ]; then
              echo "::error::Diffy comparison did not complete within ${DIFFY_MAX_WAIT}s."
              exit 1
            fi

            status="$(/tmp/diffy.phar diff:get-status "${diff_id}" || echo "0")"

            if [ "${status}" = "1" ]; then
              echo "[poll #${attempt} | ${elapsed}s] Comparison complete."
              break
            fi

            echo "[poll #${attempt} | ${elapsed}s] Comparison in progress, sleeping ${DIFFY_POLL_INTERVAL}s..."
            sleep "${DIFFY_POLL_INTERVAL}"
          done
        env:
          STEPS_COMPARE_OUTPUTS_DIFF_ID: ${{ steps.compare.outputs.diff_id }}

      - name: Fetch comparison result
        id: result
        if: steps.gate.outputs.skipped != 'true'
        run: |
          set -euo pipefail
          diff_id="${STEPS_COMPARE_OUTPUTS_DIFF_ID}"

          changes_percent="$(/tmp/diffy.phar diff:get-changes-percent "${diff_id}")"
          result_json="$(/tmp/diffy.phar diff:get-result "${diff_id}" --format=json)"

          shared_url="$(printf '%s' "${result_json}" | jq -r '.diffSharedUrl // empty')"
          pages_changed="$(printf '%s' "${result_json}" | jq -r '[.diffs | to_entries[] | select(.value | to_entries | map(.value.percentageChanges // 0) | add > 0)] | length')"
          pages_total="$(printf '%s' "${result_json}" | jq -r '.diffs | length')"

          {
            echo "changes_percent=${changes_percent}"
            echo "shared_url=${shared_url}"
            echo "pages_changed=${pages_changed}"
            echo "pages_total=${pages_total}"
          } >> "$GITHUB_OUTPUT"

          echo "Diff summary: ${pages_changed} of ${pages_total} pages changed (${changes_percent}% total)."
          echo "Report URL: ${shared_url}"
        env:
          STEPS_COMPARE_OUTPUTS_DIFF_ID: ${{ steps.compare.outputs.diff_id }}

  vr-report:
    runs-on: ubuntu-latest
    needs: vr-compare
    if: needs.vr-compare.result == 'success' && needs.vr-compare.outputs.skipped != 'true' && needs.vr-compare.outputs.pr_number != ''
    permissions:
      contents: read
      pull-requests: write

    steps:
      - name: Post sticky PR comment
        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
        with:
          number: ${{ needs.vr-compare.outputs.pr_number }}
          header: vr-diffy
          message: |
            ### Visual regression report

            - **Pages changed**: ${{ needs.vr-compare.outputs.pages_changed }} of ${{ needs.vr-compare.outputs.pages_total }}
            - **Overall difference**: ${{ needs.vr-compare.outputs.changes_percent }}%
            - **Target environment**: ${{ env.TARGET_URL }}
            - **Source environment**: ${{ env.SOURCE_ENV }}

            [View full Diffy report](${{ needs.vr-compare.outputs.shared_url }})
          hide_and_recreate: true