Skip to content

Commit aa4e8b1

Browse files
authored
Add PR comment for visual diff (#1185)
1 parent bfcc26d commit aa4e8b1

4 files changed

Lines changed: 313 additions & 1 deletion

File tree

.github/workflows/deploy-preview.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ jobs:
120120
needs: build
121121
if: ${{ github.repository == 'PaloAltoNetworks/docusaurus-openapi-docs' && !failure() && !cancelled() }}
122122
runs-on: ubuntu-latest
123+
outputs:
124+
preview_url: ${{ steps.deploy_preview.outputs.details_url }}
123125

124126
steps:
125127
- name: Checkout repository
@@ -149,3 +151,61 @@ jobs:
149151
channelId: "pr${{ github.event.number }}"
150152
env:
151153
FIREBASE_CLI_PREVIEWS: hostingchannels
154+
155+
visual_diff:
156+
name: Visual Diff
157+
needs: deploy
158+
if: ${{ github.repository == 'PaloAltoNetworks/docusaurus-openapi-docs' && !failure() && !cancelled() }}
159+
runs-on: ubuntu-latest
160+
161+
steps:
162+
- name: Checkout repository
163+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
164+
with:
165+
ref: ${{ github.event.pull_request.head.sha }}
166+
167+
- name: Setup node
168+
uses: actions/setup-node@cdca7365b2dadb8aad0a33bc7601856ffabcc48e # v4
169+
with:
170+
node-version: "20"
171+
cache: "yarn"
172+
173+
- name: Install dependencies
174+
run: yarn --prefer-offline
175+
176+
- name: Install Playwright
177+
run: npx playwright install --with-deps chromium
178+
179+
- name: Run visual diff
180+
run: ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json
181+
182+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
183+
if: always()
184+
with:
185+
name: visual_diffs
186+
path: visual_diffs
187+
188+
- name: Comment PR with results
189+
if: github.event_name == 'pull_request'
190+
uses: actions/github-script@v7
191+
with:
192+
github-token: ${{ secrets.GITHUB_TOKEN }}
193+
script: |
194+
const fs = require('fs');
195+
const results = JSON.parse(fs.readFileSync('visual_diffs/results.json', 'utf8'));
196+
const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
197+
let body = `### Visual Diff Summary\n\n[View Logs](${runUrl})\n\n`;
198+
body += `Total: ${results.summary.total}, Matches: ${results.summary.matches}, Diffs: ${results.summary.mismatches}, Skipped: ${results.summary.skipped}\n\n`;
199+
if (results.pages.length) {
200+
body += '| Page | Status |\n| --- | --- |\n';
201+
for (const p of results.pages) {
202+
if (p.status !== 'match') {
203+
body += `| ${p.path} | ${p.status} |\n`;
204+
}
205+
}
206+
}
207+
await github.rest.issues.createComment({
208+
...context.repo,
209+
issue_number: context.issue.number,
210+
body
211+
});

package.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,11 @@
7272
"start-server-and-test": "^1.14.0",
7373
"ts-jest": "^27.0.6",
7474
"ts-node": "^10.9.2",
75-
"typescript": "^5.1"
75+
"typescript": "^5.1",
76+
"fast-xml-parser": "^4.5.1",
77+
"playwright": "^1.40.0",
78+
"pixelmatch": "^5.3.0",
79+
"pngjs": "^7.0.0"
7680
},
7781
"lint-staged": {
7882
"**/*": "prettier --write --ignore-unknown"

scripts/sitemap-visual-diff.ts

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
#!/usr/bin/env ts-node
2+
/* ============================================================================
3+
* Copyright (c) Palo Alto Networks
4+
*
5+
* This source code is licensed under the MIT license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
* ========================================================================== */
8+
9+
import fs from "fs";
10+
import path from "path";
11+
12+
import { XMLParser } from "fast-xml-parser";
13+
import pixelmatch from "pixelmatch";
14+
import { chromium } from "playwright";
15+
import { PNG } from "pngjs";
16+
17+
interface Options {
18+
previewUrl: string;
19+
outputDir: string;
20+
tolerance: number;
21+
width: number;
22+
viewHeight: number;
23+
summaryFile: string;
24+
}
25+
26+
function parseArgs(): Options {
27+
const args = process.argv.slice(2);
28+
const opts: Options = {
29+
previewUrl: "",
30+
outputDir: "visual_diffs",
31+
tolerance: 0,
32+
width: 1280,
33+
viewHeight: 1024,
34+
summaryFile: "visual_diffs/results.json",
35+
};
36+
for (let i = 0; i < args.length; i++) {
37+
const arg = args[i];
38+
switch (arg) {
39+
case "-p":
40+
case "--preview-url":
41+
opts.previewUrl = args[++i];
42+
break;
43+
case "-o":
44+
case "--output-dir":
45+
opts.outputDir = args[++i];
46+
break;
47+
case "-t":
48+
case "--tolerance":
49+
opts.tolerance = Number(args[++i]);
50+
break;
51+
case "-w":
52+
case "--width":
53+
opts.width = Number(args[++i]);
54+
break;
55+
case "-v":
56+
case "--view-height":
57+
opts.viewHeight = Number(args[++i]);
58+
break;
59+
case "-s":
60+
case "--summary-file":
61+
opts.summaryFile = args[++i];
62+
break;
63+
}
64+
}
65+
return opts;
66+
}
67+
68+
async function fetchSitemap(url: string): Promise<string> {
69+
const resp = await fetch(url);
70+
if (!resp.ok) throw new Error(`Failed to fetch ${url}: ${resp.status}`);
71+
return resp.text();
72+
}
73+
74+
function parseUrlsFromSitemap(xml: string): string[] {
75+
const parser = new XMLParser();
76+
const result = parser.parse(xml);
77+
const urls = result.urlset?.url || [];
78+
const arr = Array.isArray(urls) ? urls : [urls];
79+
return arr.map((u: any) => String(u.loc).trim()).filter(Boolean);
80+
}
81+
82+
async function screenshotFullPage(page: any, url: string, outputPath: string) {
83+
await page.goto(url, { waitUntil: "networkidle" });
84+
await page.evaluate(() => {
85+
document.querySelectorAll("details").forEach((d) => {
86+
const summary = d.querySelector("summary");
87+
if (!d.open && summary) (summary as HTMLElement).click();
88+
(d as HTMLDetailsElement).open = true;
89+
d.setAttribute("data-collapsed", "false");
90+
});
91+
});
92+
await page.waitForTimeout(500);
93+
await fs.promises.mkdir(path.dirname(outputPath), { recursive: true });
94+
await page.screenshot({ path: outputPath, fullPage: true });
95+
}
96+
97+
function compareImages(
98+
prodPath: string,
99+
prevPath: string,
100+
diffPath: string,
101+
tolerance: number
102+
): boolean {
103+
const prod = PNG.sync.read(fs.readFileSync(prodPath));
104+
const prev = PNG.sync.read(fs.readFileSync(prevPath));
105+
if (prod.width !== prev.width || prod.height !== prev.height) {
106+
console.warn(`Size mismatch for ${prevPath}`);
107+
return false;
108+
}
109+
const diff = new PNG({ width: prod.width, height: prod.height });
110+
const numDiff = pixelmatch(
111+
prod.data,
112+
prev.data,
113+
diff.data,
114+
prod.width,
115+
prod.height,
116+
{
117+
threshold: tolerance,
118+
}
119+
);
120+
if (numDiff > 0) {
121+
fs.mkdirSync(path.dirname(diffPath), { recursive: true });
122+
fs.writeFileSync(diffPath, PNG.sync.write(diff));
123+
return false;
124+
}
125+
return true;
126+
}
127+
128+
async function run() {
129+
const opts = parseArgs();
130+
if (!opts.previewUrl) {
131+
throw new Error("Missing preview URL");
132+
}
133+
if (!opts.previewUrl.endsWith("/")) opts.previewUrl += "/";
134+
135+
const sitemapXml = await fetchSitemap("https://pan.dev/sitemap.xml");
136+
const paths = parseUrlsFromSitemap(await sitemapXml);
137+
console.log(`Found ${paths.length} paths.`);
138+
139+
const browser = await chromium.launch();
140+
const context = await browser.newContext({
141+
viewport: { width: opts.width, height: opts.viewHeight },
142+
});
143+
const page = await context.newPage();
144+
145+
let total = 0;
146+
let matches = 0;
147+
let mismatches = 0;
148+
let skipped = 0;
149+
const pages: { path: string; status: string }[] = [];
150+
151+
for (const url of paths) {
152+
total += 1;
153+
const cleanPath =
154+
new URL(url).pathname.replace(/^\//, "").replace(/\/$/, "") || "root";
155+
const prodSnap = path.join(opts.outputDir, "prod", `${cleanPath}.png`);
156+
const prevSnap = path.join(opts.outputDir, "preview", `${cleanPath}.png`);
157+
const diffImg = path.join(opts.outputDir, "diff", `${cleanPath}.png`);
158+
try {
159+
await screenshotFullPage(page, url, prodSnap);
160+
await screenshotFullPage(
161+
page,
162+
new URL(cleanPath, opts.previewUrl).toString(),
163+
prevSnap
164+
);
165+
if (compareImages(prodSnap, prevSnap, diffImg, opts.tolerance)) {
166+
console.log(`MATCH: /${cleanPath}`);
167+
matches += 1;
168+
pages.push({ path: `/${cleanPath}`, status: "match" });
169+
} else {
170+
console.warn(`DIFF: /${cleanPath}`);
171+
mismatches += 1;
172+
pages.push({ path: `/${cleanPath}`, status: "diff" });
173+
}
174+
} catch (e) {
175+
console.warn(`SKIP: /${cleanPath} - ${e}`);
176+
skipped += 1;
177+
pages.push({ path: `/${cleanPath}`, status: "skip" });
178+
}
179+
}
180+
181+
await browser.close();
182+
console.log(
183+
`Total: ${total}, Matches: ${matches}, Diffs: ${mismatches}, Skipped: ${skipped}`
184+
);
185+
186+
await fs.promises.mkdir(path.dirname(opts.summaryFile), { recursive: true });
187+
await fs.promises.writeFile(
188+
opts.summaryFile,
189+
JSON.stringify(
190+
{ summary: { total, matches, mismatches, skipped }, pages },
191+
null,
192+
2
193+
)
194+
);
195+
}
196+
197+
run().catch((e) => {
198+
console.error(e);
199+
process.exit(1);
200+
});

yarn.lock

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8465,6 +8465,13 @@ fast-uri@^3.0.1:
84658465
resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.0.6.tgz#88f130b77cfaea2378d56bf970dea21257a68748"
84668466
integrity sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==
84678467

8468+
fast-xml-parser@^4.5.1:
8469+
version "4.5.3"
8470+
resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.3.tgz#c54d6b35aa0f23dc1ea60b6c884340c006dc6efb"
8471+
integrity sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==
8472+
dependencies:
8473+
strnum "^1.1.1"
8474+
84688475
fastq@^1.6.0:
84698476
version "1.19.1"
84708477
resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.19.1.tgz#d50eaba803c8846a883c16492821ebcd2cda55f5"
@@ -8782,6 +8789,11 @@ fs.realpath@^1.0.0:
87828789
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
87838790
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
87848791

8792+
fsevents@2.3.2:
8793+
version "2.3.2"
8794+
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
8795+
integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
8796+
87858797
fsevents@^2.3.2, fsevents@~2.3.2:
87868798
version "2.3.3"
87878799
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -14149,6 +14161,13 @@ pixelmatch@^4.0.2:
1414914161
dependencies:
1415014162
pngjs "^3.0.0"
1415114163

14164+
pixelmatch@^5.3.0:
14165+
version "5.3.0"
14166+
resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a"
14167+
integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==
14168+
dependencies:
14169+
pngjs "^6.0.0"
14170+
1415214171
pkg-dir@^4.2.0:
1415314172
version "4.2.0"
1415414173
resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3"
@@ -14163,6 +14182,20 @@ pkg-dir@^7.0.0:
1416314182
dependencies:
1416414183
find-up "^6.3.0"
1416514184

14185+
playwright-core@1.53.2:
14186+
version "1.53.2"
14187+
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.53.2.tgz#78f71e2f727713daa8d360dc11c460022c13cf91"
14188+
integrity sha512-ox/OytMy+2w1jcYEYlOo1Hhp8hZkLCximMTUTMBXjGUA1KoFfiSZ+DU+3a739jsPY0yoKH2TFy9S2fsJas8yAw==
14189+
14190+
playwright@^1.40.0:
14191+
version "1.53.2"
14192+
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.53.2.tgz#cc2ef4a22da1ae562e0ed91edb9e22a7c4371305"
14193+
integrity sha512-6K/qQxVFuVQhRQhFsVZ9fGeatxirtrpPgxzBYWyZLEXJzqYwuL4fuNmfOfD5et1tJE4GScKyPNeLhZeRwuTU3A==
14194+
dependencies:
14195+
playwright-core "1.53.2"
14196+
optionalDependencies:
14197+
fsevents "2.3.2"
14198+
1416614199
please-upgrade-node@^3.2.0:
1416714200
version "3.2.0"
1416814201
resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942"
@@ -14180,6 +14213,16 @@ pngjs@^3.0.0, pngjs@^3.3.3:
1418014213
resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
1418114214
integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
1418214215

14216+
pngjs@^6.0.0:
14217+
version "6.0.0"
14218+
resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821"
14219+
integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==
14220+
14221+
pngjs@^7.0.0:
14222+
version "7.0.0"
14223+
resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26"
14224+
integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==
14225+
1418314226
possible-typed-array-names@^1.0.0:
1418414227
version "1.1.0"
1418514228
resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae"
@@ -16857,6 +16900,11 @@ strip-json-comments@~2.0.1:
1685716900
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
1685816901
integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==
1685916902

16903+
strnum@^1.1.1:
16904+
version "1.1.2"
16905+
resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.1.2.tgz#57bca4fbaa6f271081715dbc9ed7cee5493e28e4"
16906+
integrity sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==
16907+
1686016908
strong-log-transformer@2.1.0:
1686116909
version "2.1.0"
1686216910
resolved "https://registry.yarnpkg.com/strong-log-transformer/-/strong-log-transformer-2.1.0.tgz#0f5ed78d325e0421ac6f90f7f10e691d6ae3ae10"

0 commit comments

Comments
 (0)