PaloAltoNetworks · sserrata · Jul 1, 2025 · Jul 1, 2025 · Jul 1, 2025 · Jul 1, 2025
diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml
@@ -179,6 +179,9 @@ jobs:
       - name: Run visual diff
         run: yarn ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json --concurrency 4 --paths "/tests/"
 
+      - name: Generate report and summary
+        run: yarn ts-node scripts/generate-visual-diff-report.ts visual_diffs/results.json visual_diffs/index.html
+
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()
         with:

diff --git a/scripts/generate-visual-diff-report.ts b/scripts/generate-visual-diff-report.ts
@@ -0,0 +1,150 @@
+#!/usr/bin/env ts-node
+/* ============================================================================
+ * Copyright (c) Palo Alto Networks
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ * ========================================================================== */
+
+import fs from "fs";
+import path from "path";
+
+interface Summary {
+  total: number;
+  matches: number;
+  mismatches: number;
+  skipped: number;
+}
+
+interface Page {
+  path: string;
+  status: string;
+}
+
+function clean(p: string): string {
+  return p.replace(/^\//, "").replace(/\/$/, "") || "root";
+}
+
+function generateHTML(results: { summary: Summary; pages: Page[] }): string {
+  const pages = results.pages.map((p) => ({ ...p, clean: clean(p.path) }));
+  const listItems = pages
+    .map((p, i) => {
+      return `<li><a href="#" data-index="${i}" class="status-${p.status}">${p.path}</a></li>`;
+    })
+    .join("\n");
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="UTF-8" />
+<title>Visual Diff Report</title>
+<style>
+body { margin:0; display:flex; height:100vh; font-family: Arial, sans-serif; }
+#sidebar { width:250px; background:#f7f7f7; overflow-y:auto; border-right:1px solid #ddd; padding:10px; }
+#content { flex:1; overflow:auto; padding:10px; }
+#diffImages { display:flex; gap:10px; margin-top:1em; }
+#diffImages img { border:1px solid #ccc; max-width:100%; background:#000; }
+.status-diff { color:#d33; }
+.status-match { color:#090; }
+.status-skip { color:#999; }
+</style>
+</head>
+<body>
+<div id="sidebar">
+<h2>Pages</h2>
+<ul>${listItems}</ul>
+</div>
+<div id="content">
+<h2 id="title">Select a page</h2>
+<div id="diffImages" style="display:none">
+  <div><div>Prod</div><img id="img-prod" /></div>
+  <div><div>Preview</div><img id="img-prev" /></div>
+  <div><div>Diff</div><img id="img-diff" /></div>
+</div>
+</div>
+<script>
+const pages = ${JSON.stringify(pages)};
+function show(i){
+  const p = pages[i];
+  if(!p) return;
+  document.getElementById('title').textContent = p.path + ' (' + p.status + ')';
+  document.getElementById('img-prod').src = 'prod/' + p.clean + '.png';
+  document.getElementById('img-prev').src = 'preview/' + p.clean + '.png';
+  document.getElementById('img-diff').src = 'diff/' + p.clean + '.png';
+  document.getElementById('diffImages').style.display = 'flex';
+}
+document.querySelectorAll('#sidebar a').forEach((a) => {
+  a.addEventListener('click', function(e){
+    e.preventDefault();
+    show(this.getAttribute('data-index'));
+  });
+});
+if(pages.length) show(0);
+</script>
+</body>
+</html>`;
+}
+
+function encodeImage(filePath: string): string {
+  try {
+    const data = fs.readFileSync(filePath);
+    return `data:image/png;base64,${data.toString("base64")}`;
+  } catch {
+    return "";
+  }
+}
+
+function generateMarkdown(
+  results: { summary: Summary; pages: Page[] },
+  baseDir: string
+): string {
+  const lines: string[] = [];
+  lines.push("### Visual Diff Summary\n");
+  lines.push(
+    `Total: ${results.summary.total}, Matches: ${results.summary.matches}, Diffs: ${results.summary.mismatches}, Skipped: ${results.summary.skipped}\n`
+  );
+  if (results.pages.length) {
+    lines.push("| Page | Status | Prod | Preview | Diff |");
+    lines.push("| --- | --- | --- | --- | --- |");
+    for (const p of results.pages) {
+      const cleanPath = clean(p.path);
+      if (p.status === "diff") {
+        const prod = encodeImage(
+          path.join(baseDir, "prod", `${cleanPath}.png`)
+        );
+        const prev = encodeImage(
+          path.join(baseDir, "preview", `${cleanPath}.png`)
+        );
+        const diff = encodeImage(
+          path.join(baseDir, "diff", `${cleanPath}.png`)
+        );
+        lines.push(
+          `| ${p.path} | <span style="color:#d33">diff</span> | ![](${prod}) | ![](${prev}) | ![](${diff}) |`
+        );
+      } else {
+        const color = p.status === "match" ? "#090" : "#999";
+        lines.push(
+          `| ${p.path} | <span style="color:${color}">${p.status}</span> | | | |`
+        );
+      }
+    }
+  }
+  lines.push("");
+  lines.push("[Download full report](./visual_diffs/index.html)\n");
+  return lines.join("\n");
+}
+
+function main() {
+  const input = process.argv[2] || path.join("visual_diffs", "results.json");
+  const output = process.argv[3] || path.join("visual_diffs", "index.html");
+  const results = JSON.parse(fs.readFileSync(input, "utf8"));
+  const html = generateHTML(results);
+  fs.writeFileSync(output, html);
+
+  const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+  if (summaryPath) {
+    const markdown = generateMarkdown(results, path.dirname(output));
+    fs.appendFileSync(summaryPath, `${markdown}\n`);
+  }
+}
+
+main();
diff --git a/scripts/sitemap-visual-diff.ts b/scripts/sitemap-visual-diff.ts
@@ -31,7 +31,7 @@ function parseArgs(): Options {
   const opts: Options = {
     previewUrl: "",
     outputDir: "visual_diffs",
-    tolerance: 0,
+    tolerance: 0.3,
     width: 1280,
     viewHeight: 1024,
     concurrency: 4,
@@ -100,16 +100,31 @@ function parseUrlsFromSitemap(xml: string): string[] {
 async function screenshotFullPage(page: any, url: string, outputPath: string) {
   await page.goto(url, { waitUntil: "networkidle" });
   await page.evaluate(() => {
-    document.querySelectorAll("details").forEach((d) => {
-      const summary = d.querySelector("summary");
-      if (!d.open && summary) (summary as HTMLElement).click();
-      (d as HTMLDetailsElement).open = true;
-      d.setAttribute("data-collapsed", "false");
+    document.querySelectorAll("div.container details").forEach((el) => {
+      const detail = el as HTMLDetailsElement;
+      const summary = detail.querySelector("summary");
+      if (!detail.open && summary) (summary as HTMLElement).click();
+      detail.open = true;
+      detail.setAttribute("data-collapsed", "false");
     });
   });
   await page.waitForTimeout(500);
   await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });
-  await page.screenshot({ path: outputPath, fullPage: true });
+  const container = await page.$("div.container");
+  if (container) {
+    await container.screenshot({ path: outputPath });
+  } else {
+    await page.screenshot({ path: outputPath, fullPage: true });
+  }
+}
+
+function padImage(img: PNG, width: number, height: number): PNG {
+  if (img.width === width && img.height === height) {
+    return img;
+  }
+  const out = new PNG({ width, height });
+  PNG.bitblt(img, out, 0, 0, img.width, img.height, 0, 0);
+  return out;
 }
 
 function compareImages(
@@ -119,11 +134,16 @@ function compareImages(
   tolerance: number,
   diffAlpha: number
 ): boolean {
-  const prod = PNG.sync.read(fs.readFileSync(prodPath));
-  const prev = PNG.sync.read(fs.readFileSync(prevPath));
+  let prod = PNG.sync.read(fs.readFileSync(prodPath));
+  let prev = PNG.sync.read(fs.readFileSync(prevPath));
   if (prod.width !== prev.width || prod.height !== prev.height) {
-    console.warn(`Size mismatch for ${prevPath}`);
-    return false;
+    const width = Math.max(prod.width, prev.width);
+    const height = Math.max(prod.height, prev.height);
+    console.warn(
+      `Size mismatch for ${prevPath}, padding images to ${width}x${height}`
+    );
+    prod = padImage(prod, width, height);
+    prev = padImage(prev, width, height);
   }
   const diff = new PNG({ width: prod.width, height: prod.height });
   const numDiff = pixelmatch(