diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index f0405b89c..26e330d22 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -179,6 +179,9 @@ jobs: - name: Run visual diff run: yarn ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json --concurrency 4 --paths "/tests/" + - name: Generate report and summary + run: yarn ts-node scripts/generate-visual-diff-report.ts visual_diffs/results.json visual_diffs/index.html + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: diff --git a/scripts/generate-visual-diff-report.ts b/scripts/generate-visual-diff-report.ts new file mode 100644 index 000000000..24829de9d --- /dev/null +++ b/scripts/generate-visual-diff-report.ts @@ -0,0 +1,150 @@ +#!/usr/bin/env ts-node +/* ============================================================================ + * Copyright (c) Palo Alto Networks + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * ========================================================================== */ + +import fs from "fs"; +import path from "path"; + +interface Summary { + total: number; + matches: number; + mismatches: number; + skipped: number; +} + +interface Page { + path: string; + status: string; +} + +function clean(p: string): string { + return p.replace(/^\//, "").replace(/\/$/, "") || "root"; +} + +function generateHTML(results: { summary: Summary; pages: Page[] }): string { + const pages = results.pages.map((p) => ({ ...p, clean: clean(p.path) })); + const listItems = pages + .map((p, i) => { + return `
  • ${p.path}
    • `; + }) + .join("\n"); + return ` + + + +Visual Diff Report + + + + +
    +

    Select a page

    + +
    + + +`; +} + +function encodeImage(filePath: string): string { + try { + const data = fs.readFileSync(filePath); + return `data:image/png;base64,${data.toString("base64")}`; + } catch { + return ""; + } +} + +function generateMarkdown( + results: { summary: Summary; pages: Page[] }, + baseDir: string +): string { + const lines: string[] = []; + lines.push("### Visual Diff Summary\n"); + lines.push( + `Total: ${results.summary.total}, Matches: ${results.summary.matches}, Diffs: ${results.summary.mismatches}, Skipped: ${results.summary.skipped}\n` + ); + if (results.pages.length) { + lines.push("| Page | Status | Prod | Preview | Diff |"); + lines.push("| --- | --- | --- | --- | --- |"); + for (const p of results.pages) { + const cleanPath = clean(p.path); + if (p.status === "diff") { + const prod = encodeImage( + path.join(baseDir, "prod", `${cleanPath}.png`) + ); + const prev = encodeImage( + path.join(baseDir, "preview", `${cleanPath}.png`) + ); + const diff = encodeImage( + path.join(baseDir, "diff", `${cleanPath}.png`) + ); + lines.push( + `| ${p.path} | diff | ![](${prod}) | ![](${prev}) | ![](${diff}) |` + ); + } else { + const color = p.status === "match" ? "#090" : "#999"; + lines.push( + `| ${p.path} | ${p.status} | | | |` + ); + } + } + } + lines.push(""); + lines.push("[Download full report](./visual_diffs/index.html)\n"); + return lines.join("\n"); +} + +function main() { + const input = process.argv[2] || path.join("visual_diffs", "results.json"); + const output = process.argv[3] || path.join("visual_diffs", "index.html"); + const results = JSON.parse(fs.readFileSync(input, "utf8")); + const html = generateHTML(results); + fs.writeFileSync(output, html); + + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) { + const markdown = generateMarkdown(results, path.dirname(output)); + fs.appendFileSync(summaryPath, `${markdown}\n`); + } +} + +main(); diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts index 9a60ae153..67a403410 100644 --- a/scripts/sitemap-visual-diff.ts +++ b/scripts/sitemap-visual-diff.ts @@ -31,7 +31,7 @@ function parseArgs(): Options { const opts: Options = { previewUrl: "", outputDir: "visual_diffs", - tolerance: 0, + tolerance: 0.3, width: 1280, viewHeight: 1024, concurrency: 4, @@ -100,16 +100,31 @@ function parseUrlsFromSitemap(xml: string): string[] { async function screenshotFullPage(page: any, url: string, outputPath: string) { await page.goto(url, { waitUntil: "networkidle" }); await page.evaluate(() => { - document.querySelectorAll("details").forEach((d) => { - const summary = d.querySelector("summary"); - if (!d.open && summary) (summary as HTMLElement).click(); - (d as HTMLDetailsElement).open = true; - d.setAttribute("data-collapsed", "false"); + document.querySelectorAll("div.container details").forEach((el) => { + const detail = el as HTMLDetailsElement; + const summary = detail.querySelector("summary"); + if (!detail.open && summary) (summary as HTMLElement).click(); + detail.open = true; + detail.setAttribute("data-collapsed", "false"); }); }); await page.waitForTimeout(500); await fs.promises.mkdir(path.dirname(outputPath), { recursive: true }); - await page.screenshot({ path: outputPath, fullPage: true }); + const container = await page.$("div.container"); + if (container) { + await container.screenshot({ path: outputPath }); + } else { + await page.screenshot({ path: outputPath, fullPage: true }); + } +} + +function padImage(img: PNG, width: number, height: number): PNG { + if (img.width === width && img.height === height) { + return img; + } + const out = new PNG({ width, height }); + PNG.bitblt(img, out, 0, 0, img.width, img.height, 0, 0); + return out; } function compareImages( @@ -119,11 +134,16 @@ function compareImages( tolerance: number, diffAlpha: number ): boolean { - const prod = PNG.sync.read(fs.readFileSync(prodPath)); - const prev = PNG.sync.read(fs.readFileSync(prevPath)); + let prod = PNG.sync.read(fs.readFileSync(prodPath)); + let prev = PNG.sync.read(fs.readFileSync(prevPath)); if (prod.width !== prev.width || prod.height !== prev.height) { - console.warn(`Size mismatch for ${prevPath}`); - return false; + const width = Math.max(prod.width, prev.width); + const height = Math.max(prod.height, prev.height); + console.warn( + `Size mismatch for ${prevPath}, padding images to ${width}x${height}` + ); + prod = padImage(prod, width, height); + prev = padImage(prev, width, height); } const diff = new PNG({ width: prod.width, height: prod.height }); const numDiff = pixelmatch(