From cb7d1c04cd127b6d187cb716b92e0e8e177ab2e0 Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:01:50 -0500 Subject: [PATCH 1/6] chore: raise diff tolerance --- .github/workflows/deploy-preview.yml | 3 + scripts/generate-visual-diff-report.ts | 150 +++++++++++++++++++++++++ scripts/sitemap-visual-diff.ts | 2 +- 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 scripts/generate-visual-diff-report.ts diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index f0405b89c..26e330d22 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -179,6 +179,9 @@ jobs: - name: Run visual diff run: yarn ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json --concurrency 4 --paths "/tests/" + - name: Generate report and summary + run: yarn ts-node scripts/generate-visual-diff-report.ts visual_diffs/results.json visual_diffs/index.html + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: diff --git a/scripts/generate-visual-diff-report.ts b/scripts/generate-visual-diff-report.ts new file mode 100644 index 000000000..24829de9d --- /dev/null +++ b/scripts/generate-visual-diff-report.ts @@ -0,0 +1,150 @@ +#!/usr/bin/env ts-node +/* ============================================================================ + * Copyright (c) Palo Alto Networks + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * ========================================================================== */ + +import fs from "fs"; +import path from "path"; + +interface Summary { + total: number; + matches: number; + mismatches: number; + skipped: number; +} + +interface Page { + path: string; + status: string; +} + +function clean(p: string): string { + return p.replace(/^\//, "").replace(/\/$/, "") || "root"; +} + +function generateHTML(results: { summary: Summary; pages: Page[] }): string { + const pages = results.pages.map((p) => ({ ...p, clean: clean(p.path) })); + const listItems = pages + .map((p, i) => { + return `
  • ${p.path}
    • `; + }) + .join("\n"); + return ` + + + +Visual Diff Report + + + + +
    +

    Select a page

    + +
    + + +`; +} + +function encodeImage(filePath: string): string { + try { + const data = fs.readFileSync(filePath); + return `data:image/png;base64,${data.toString("base64")}`; + } catch { + return ""; + } +} + +function generateMarkdown( + results: { summary: Summary; pages: Page[] }, + baseDir: string +): string { + const lines: string[] = []; + lines.push("### Visual Diff Summary\n"); + lines.push( + `Total: ${results.summary.total}, Matches: ${results.summary.matches}, Diffs: ${results.summary.mismatches}, Skipped: ${results.summary.skipped}\n` + ); + if (results.pages.length) { + lines.push("| Page | Status | Prod | Preview | Diff |"); + lines.push("| --- | --- | --- | --- | --- |"); + for (const p of results.pages) { + const cleanPath = clean(p.path); + if (p.status === "diff") { + const prod = encodeImage( + path.join(baseDir, "prod", `${cleanPath}.png`) + ); + const prev = encodeImage( + path.join(baseDir, "preview", `${cleanPath}.png`) + ); + const diff = encodeImage( + path.join(baseDir, "diff", `${cleanPath}.png`) + ); + lines.push( + `| ${p.path} | diff | ![](${prod}) | ![](${prev}) | ![](${diff}) |` + ); + } else { + const color = p.status === "match" ? "#090" : "#999"; + lines.push( + `| ${p.path} | ${p.status} | | | |` + ); + } + } + } + lines.push(""); + lines.push("[Download full report](./visual_diffs/index.html)\n"); + return lines.join("\n"); +} + +function main() { + const input = process.argv[2] || path.join("visual_diffs", "results.json"); + const output = process.argv[3] || path.join("visual_diffs", "index.html"); + const results = JSON.parse(fs.readFileSync(input, "utf8")); + const html = generateHTML(results); + fs.writeFileSync(output, html); + + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) { + const markdown = generateMarkdown(results, path.dirname(output)); + fs.appendFileSync(summaryPath, `${markdown}\n`); + } +} + +main(); diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index 9a60ae153..cb484dc50 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -31,7 +31,7 @@ function parseArgs(): Options { const opts: Options = { previewUrl: "", outputDir: "visual_diffs", - tolerance: 0, + tolerance: 0.2, width: 1280, viewHeight: 1024, concurrency: 4, From 67372c80b60f8099c5fa6f6017a2adff971f3272 Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:04:38 -0500 Subject: [PATCH 2/6] Limit details expansion to container --- scripts/sitemap-visual-diff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index cb484dc50..ac4eb0391 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -100,7 +100,7 @@ function parseUrlsFromSitemap(xml: string): string[] { async function screenshotFullPage(page: any, url: string, outputPath: string) { await page.goto(url, { waitUntil: "networkidle" }); await page.evaluate(() => { - document.querySelectorAll("details").forEach((d) => { + document.querySelectorAll("div.container details").forEach((d) => { const summary = d.querySelector("summary"); if (!d.open && summary) (summary as HTMLElement).click(); (d as HTMLDetailsElement).open = true; From 56a8d65e7e886b71620bbc86441abd3eb2d9ec77 Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 13:41:18 -0500 Subject: [PATCH 3/6] Fix TypeScript error when expanding details --- scripts/sitemap-visual-diff.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index ac4eb0391..9c4790a35 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -100,11 +100,12 @@ function parseUrlsFromSitemap(xml: string): string[] { async function screenshotFullPage(page: any, url: string, outputPath: string) { await page.goto(url, { waitUntil: "networkidle" }); await page.evaluate(() => { - document.querySelectorAll("div.container details").forEach((d) => { - const summary = d.querySelector("summary"); - if (!d.open && summary) (summary as HTMLElement).click(); - (d as HTMLDetailsElement).open = true; - d.setAttribute("data-collapsed", "false"); + document.querySelectorAll("div.container details").forEach((el) => { + const detail = el as HTMLDetailsElement; + const summary = detail.querySelector("summary"); + if (!detail.open && summary) (summary as HTMLElement).click(); + detail.open = true; + detail.setAttribute("data-collapsed", "false"); }); }); await page.waitForTimeout(500); From 0b036fbb3c251e281b7470214e2e7eacfb0f9cfd Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:42:27 -0500 Subject: [PATCH 4/6] Increase default visual diff tolerance --- scripts/sitemap-visual-diff.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index 9c4790a35..9dd611b74 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -31,7 +31,7 @@ function parseArgs(): Options { const opts: Options = { previewUrl: "", outputDir: "visual_diffs", - tolerance: 0.2, + tolerance: 0.3, width: 1280, viewHeight: 1024, concurrency: 4, From 40d3a0f642cd432bcf7c1a05a29868007d8209bc Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 17:49:00 -0500 Subject: [PATCH 5/6] Capture only container when screenshotting --- scripts/sitemap-visual-diff.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index 9dd611b74..fb0c776f9 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -110,7 +110,12 @@ async function screenshotFullPage(page: any, url: string, outputPath: string) { }); await page.waitForTimeout(500); await fs.promises.mkdir(path.dirname(outputPath), { recursive: true }); - await page.screenshot({ path: outputPath, fullPage: true }); + const container = await page.$("div.container"); + if (container) { + await container.screenshot({ path: outputPath }); + } else { + await page.screenshot({ path: outputPath, fullPage: true }); + } } function compareImages( From 862b4da40fff006d1bfb9d45ad2f610076a92292 Mon Sep 17 00:00:00 2001 From: Steven Serrata <9343811+sserrata@users.noreply.github.com> Date: Tue, 1 Jul 2025 18:09:40 -0500 Subject: [PATCH 6/6] Pad mismatched screenshots --- scripts/sitemap-visual-diff.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index fb0c776f9..67a403410 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -118,6 +118,15 @@ async function screenshotFullPage(page: any, url: string, outputPath: string) { } } +function padImage(img: PNG, width: number, height: number): PNG { + if (img.width === width && img.height === height) { + return img; + } + const out = new PNG({ width, height }); + PNG.bitblt(img, out, 0, 0, img.width, img.height, 0, 0); + return out; +} + function compareImages( prodPath: string, prevPath: string, @@ -125,11 +134,16 @@ function compareImages( tolerance: number, diffAlpha: number ): boolean { - const prod = PNG.sync.read(fs.readFileSync(prodPath)); - const prev = PNG.sync.read(fs.readFileSync(prevPath)); + let prod = PNG.sync.read(fs.readFileSync(prodPath)); + let prev = PNG.sync.read(fs.readFileSync(prevPath)); if (prod.width !== prev.width || prod.height !== prev.height) { - console.warn(`Size mismatch for ${prevPath}`); - return false; + const width = Math.max(prod.width, prev.width); + const height = Math.max(prod.height, prev.height); + console.warn( + `Size mismatch for ${prevPath}, padding images to ${width}x${height}` + ); + prod = padImage(prod, width, height); + prev = padImage(prev, width, height); } const diff = new PNG({ width: prod.width, height: prod.height }); const numDiff = pixelmatch(