Merge branch 'main' into test-differ

Author: sserrata

.github/workflows/deploy-preview.yml

@@ -177,7 +177,7 @@ jobs:
         run: npx playwright install --with-deps chromium
 
       - name: Run visual diff
-        run: ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json
+        run: yarn ts-node scripts/sitemap-visual-diff.ts --preview-url ${{ needs.deploy.outputs.preview_url }} --summary-file visual_diffs/results.json --concurrency 4
 
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: always()

scripts/sitemap-visual-diff.ts

@@ -20,6 +20,7 @@ interface Options {
   tolerance: number;
   width: number;
   viewHeight: number;
+  concurrency: number;
   summaryFile: string;
 }
 
@@ -31,6 +32,7 @@ function parseArgs(): Options {
     tolerance: 0,
     width: 1280,
     viewHeight: 1024,
+    concurrency: 4,
     summaryFile: "visual_diffs/results.json",
   };
   for (let i = 0; i < args.length; i++) {
@@ -56,6 +58,10 @@ function parseArgs(): Options {
       case "--view-height":
         opts.viewHeight = Number(args[++i]);
         break;
+      case "-c":
+      case "--concurrency":
+        opts.concurrency = Number(args[++i]);
+        break;
       case "-s":
       case "--summary-file":
         opts.summaryFile = args[++i];
@@ -125,36 +131,57 @@ function compareImages(
   return true;
 }
 
+async function promisePool<T>(
+  items: T[],
+  concurrency: number,
+  iteratorFn: (item: T) => Promise<void>
+) {
+  const executing: Promise<void>[] = [];
+
+  for (const item of items) {
+    const p = iteratorFn(item).then(() => {
+      executing.splice(executing.indexOf(p), 1);
+    });
+    executing.push(p);
+    if (executing.length >= concurrency) {
+      await Promise.race(executing);
+    }
+  }
+  await Promise.all(executing);
+}
+
 async function run() {
   const opts = parseArgs();
   if (!opts.previewUrl) {
     throw new Error("Missing preview URL");
   }
   if (!opts.previewUrl.endsWith("/")) opts.previewUrl += "/";
 
-  const sitemapXml = await fetchSitemap("https://docusaurus-openapi.tryingpan.dev/sitemap.xml");
+  const sitemapXml = await fetchSitemap(
+    "https://docusaurus-openapi.tryingpan.dev/sitemap.xml"
+  );
+
   const paths = parseUrlsFromSitemap(await sitemapXml);
   console.log(`Found ${paths.length} paths.`);
 
   const browser = await chromium.launch();
   const context = await browser.newContext({
     viewport: { width: opts.width, height: opts.viewHeight },
   });
-  const page = await context.newPage();
-
   let total = 0;
   let matches = 0;
   let mismatches = 0;
   let skipped = 0;
   const pages: { path: string; status: string }[] = [];
 
-  for (const url of paths) {
+  async function processPath(url: string) {
     total += 1;
     const cleanPath =
       new URL(url).pathname.replace(/^\//, "").replace(/\/$/, "") || "root";
     const prodSnap = path.join(opts.outputDir, "prod", `${cleanPath}.png`);
     const prevSnap = path.join(opts.outputDir, "preview", `${cleanPath}.png`);
     const diffImg = path.join(opts.outputDir, "diff", `${cleanPath}.png`);
+    const page = await context.newPage();
     try {
       await screenshotFullPage(page, url, prodSnap);
       await screenshotFullPage(
@@ -176,8 +203,11 @@ async function run() {
       skipped += 1;
       pages.push({ path: `/${cleanPath}`, status: "skip" });
     }
+    await page.close();
   }
 
+  await promisePool(paths, opts.concurrency, processPath);
+
   await browser.close();
   console.log(
     `Total: ${total}, Matches: ${matches}, Diffs: ${mismatches}, Skipped: ${skipped}`