feat: automate fetching and sanitization of webpack ecosystem readmes

.gitignore

@@ -3,3 +3,5 @@ out
 *.generated.*
 /.cache
 /pages/api
+/pages/loaders/
+/pages/plugins/

package.json

@@ -2,8 +2,11 @@
   "scripts": {
     "prep": "node scripts/prepare/index.mjs",
     "build:md": "node scripts/markdown/index.mjs",
+    "build:md:loaders": "node scripts/fetch-readmes.mjs --loaders",
+    "build:md:plugins": "node scripts/fetch-readmes.mjs --plugins",
+    "build:md:readmes": "node scripts/fetch-readmes.mjs",
     "build:html": "node scripts/html/index.mjs",
-    "build": "npm run prep && npm run build:md && npm run build:html",
+    "build": "npm run prep && npm run build:md && npm run build:md:readmes && npm run build:html",
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
     "format": "prettier --write .",

scripts/fetch-readmes.mjs

@@ -0,0 +1,208 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const { GH_TOKEN } = process.env;
+if (!GH_TOKEN) {
+  throw new Error('GH_TOKEN environment variable is not set');
+}
+
+const BASE_HEADERS = {
+  Authorization: `Bearer ${GH_TOKEN}`,
+  'X-GitHub-Api-Version': '2022-11-28',
+};
+
+const parseNextLink = linkHeader => {
+  if (!linkHeader) return null;
+  const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+  return match ? match[1] : null;
+};
+
+const discoverRepos = async () => {
+  const loaders = [];
+  const plugins = [];
+  let url =
+    'https://api.github.com/orgs/webpack/repos?per_page=100&type=public';
+
+  while (url) {
+    const res = await fetch(url, { headers: BASE_HEADERS });
+    if (!res.ok)
+      throw new Error(
+        `Failed to list org repos: ${res.status} ${res.statusText}`
+      );
+
+    const repos = await res.json();
+    for (const repo of repos) {
+      if (repo.archived) continue;
+      if (repo.name.endsWith('-loader')) {
+        loaders.push(repo.full_name);
+      } else if (
+        repo.name.endsWith('-webpack-plugin') ||
+        repo.name.endsWith('-plugin')
+      ) {
+        plugins.push(repo.full_name);
+      }
+    }
+
+    url = parseNextLink(res.headers.get('link'));
+  }
+
+  return { loaders, plugins };
+};
+
+const stripLeadingDiv = content =>
+  content.replace(/^\s*<div[\s\S]*?<\/div>\n*/i, '');
+
+// Remove badge lines - lines consisting only of [![...][ref]][ref] or [![...](url)](url) links
+const stripBadges = content =>
+  content
+    .replace(
+      /^(\[!\[[^\]]*\](?:\[[^\]]*\]|\([^)]*\))\]\s*(?:\[[^\]]*\]|\([^)]*\))\s*)+$/gm,
+      ''
+    )
+    .replace(/\n{3,}/g, '\n\n');
+
+const SUPPORTED_LANGS = new Set([
+  'bash',
+  'c',
+  'c++',
+  'cjs',
+  'coffee',
+  'coffeescript',
+  'console',
+  'cpp',
+  'diff',
+  'docker',
+  'dockerfile',
+  'glsl',
+  'gql',
+  'graphql',
+  'http',
+  'ini',
+  'java',
+  'javascript',
+  'js',
+  'json',
+  'jsx',
+  'mjs',
+  'powershell',
+  'ps',
+  'ps1',
+  'regex',
+  'regexp',
+  'sh',
+  'shell',
+  'shellscript',
+  'shellsession',
+  'sql',
+  'ts',
+  'tsx',
+  'typescript',
+  'xml',
+  'yaml',
+  'yml',
+  'zsh',
+]);
+
+const sanitizeCodeFences = content =>
+  content.replace(/^```([a-zA-Z0-9_+-]+)\b/gm, (match, lang) =>
+    SUPPORTED_LANGS.has(lang.toLowerCase()) ? match : '```'
+  );
+
+// remark-gfm does not support GitHub alert syntax (> [!TYPE]); rewrite to bold label inside the blockquote.
+const GFM_ALERT_LABELS = {
+  NOTE: 'Note',
+  TIP: 'Tip',
+  IMPORTANT: 'Important',
+  WARNING: 'Warning',
+  CAUTION: 'Caution',
+};
+const GFM_ALERT_RE =
+  /^([ \t]*>[ \t]*)\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ \t]*$/gim;
+
+const transformGfmAlerts = content =>
+  content.replace(
+    GFM_ALERT_RE,
+    (_, prefix, type) => `${prefix}**${GFM_ALERT_LABELS[type]}:**`
+  );
+
+const processContent = content =>
+  transformGfmAlerts(sanitizeCodeFences(stripBadges(stripLeadingDiv(content))));
+
+const fetchReadme = async fullName => {
+  const url = `https://api.github.com/repos/${fullName}/readme`;
+  const res = await fetch(url, {
+    headers: { ...BASE_HEADERS, Accept: 'application/vnd.github.raw' },
+  });
+  return res.ok
+    ? { ok: true, text: await res.text() }
+    : { ok: false, status: res.status };
+};
+
+const processRepos = async (
+  repos,
+  { layout, groupName, basePath, outputDir }
+) => {
+  mkdirSync(outputDir, { recursive: true });
+  const repoName = r => r.split('/')[1];
+  console.log(
+    `Discovered ${groupName.toLowerCase()}: ${repos.map(repoName).join(', ')}`
+  );
+
+  const fetched = [];
+  for (const fullName of repos) {
+    const name = repoName(fullName);
+    const result = await fetchReadme(fullName);
+    if (!result.ok) {
+      console.log(`Failed: ${name} — ${result.status}`);
+      continue;
+    }
+    const content = `---\nlayout: ${layout}\n---\n\n${processContent(result.text)}`;
+    writeFileSync(join(outputDir, `${name}.md`), content, 'utf8');
+    fetched.push(name);
+    console.log(`Fetched: ${name}`);
+  }
+
+  const siteJson = {
+    sidebar: [
+      {
+        groupName,
+        items: fetched
+          .sort()
+          .map(name => ({ link: `${basePath}/${name}`, label: name })),
+      },
+    ],
+  };
+  writeFileSync(
+    join(outputDir, 'site.json'),
+    JSON.stringify(siteJson, null, 2) + '\n',
+    'utf8'
+  );
+  console.log(
+    `Written: ${outputDir}/site.json (${fetched.length} ${groupName.toLowerCase()})`
+  );
+};
+
+const args = process.argv.slice(2);
+const runLoaders = args.includes('--loaders') || args.length === 0;
+const runPlugins = args.includes('--plugins') || args.length === 0;
+
+const root = new URL('..', import.meta.url).pathname;
+const { loaders, plugins } = await discoverRepos();
+
+if (runLoaders) {
+  await processRepos(loaders, {
+    layout: 'loader',
+    groupName: 'Loaders',
+    basePath: '/loaders',
+    outputDir: join(root, 'pages/loaders'),
+  });
+}
+
+if (runPlugins) {
+  await processRepos(plugins, {
+    layout: 'plugin',
+    groupName: 'Plugins',
+    basePath: '/plugins',
+    outputDir: join(root, 'pages/plugins'),
+  });
+}