Improve markdown conversion quality (#1229)

Refactor HTML-to-Markdown converter to eliminate navigation chrome and noise artifacts, improving content quality for AI consumption.

Author: GBKS

netlify/edge-functions/markdown-negotiation.ts

@@ -1,6 +1,34 @@
 const MARKDOWN_CONTENT_TYPE = "text/markdown; charset=utf-8";
 const TEXT_NODE = 3;
 const ELEMENT_NODE = 1;
+const NOISE_SELECTORS = [
+  "script",
+  "style",
+  "noscript",
+  "template",
+  "svg",
+  "header",
+  "footer",
+  "nav",
+  "aside",
+  ".site-header",
+  ".site-footer",
+  ".side-bar",
+  ".secondary-nav",
+  ".search-input-wrap",
+  ".social-media-list",
+  ".footer-nav",
+  ".skip-to-content-link",
+  ".header-anchor",
+  ".next-previous",
+  ".next-previous--feedback",
+  ".home-banner-info",
+  "#secondary-nav",
+  "#site-nav",
+  "#toggled-search",
+  "[aria-label='Main navigation']",
+  "[aria-label='Footer navigation']",
+];
 
 export default async (request: Request, context: { next: () => Promise<Response> }) => {
   const response = await context.next();
@@ -76,18 +104,15 @@ function htmlToMarkdown(html: string, baseUrl: URL): string {
     return htmlToMarkdownFallback(html, baseUrl);
   }
 
-  for (const selector of ["script", "style", "noscript"]) {
-    doc.querySelectorAll(selector).forEach((node: any) => node.remove());
-  }
+  pruneDocument(doc);
 
   const title = normalizeWhitespace(doc.querySelector("title")?.textContent || "");
-  const body = doc.body;
-  const bodyMarkdown = body ? renderChildren(body, baseUrl) : "";
+  const contentRoot = selectContentRoot(doc);
+  const bodyMarkdown = contentRoot ? renderChildren(contentRoot, baseUrl) : "";
 
   const parts: string[] = [];
   if (title) {
     parts.push(`---\ntitle: ${title}\n---`);
-    parts.push(`# ${title}`);
   }
 
   if (bodyMarkdown) {
@@ -102,19 +127,20 @@ function htmlToMarkdownFallback(html: string, baseUrl: URL): string {
     .replace(/<!--[\s\S]*?-->/g, "")
     .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, "")
     .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, "")
-    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, "");
+    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, "")
+    .replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, "");
 
   const titleMatch = sanitized.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i);
   const title = titleMatch ? normalizeWhitespace(decodeHtmlEntities(stripTags(titleMatch[1]))) : "";
 
   const bodyMatch = sanitized.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
-  const bodyHtml = bodyMatch ? bodyMatch[1] : sanitized;
+  const rawBodyHtml = bodyMatch ? bodyMatch[1] : sanitized;
+  const bodyHtml = extractPrimaryHtmlFragment(rawBodyHtml);
   const bodyMarkdown = renderHtmlFragmentToMarkdown(bodyHtml, baseUrl);
 
   const parts: string[] = [];
   if (title) {
     parts.push(`---\ntitle: ${title}\n---`);
-    parts.push(`# ${title}`);
   }
 
   if (bodyMarkdown) {
@@ -126,7 +152,7 @@ function htmlToMarkdownFallback(html: string, baseUrl: URL): string {
 
 function renderHtmlFragmentToMarkdown(html: string, baseUrl: URL): string {
   const protectedBlocks: string[] = [];
-  let content = html;
+  let content = stripFallbackNoise(html);
 
   // Protect code blocks so later tag cleanup does not alter their content.
   content = content.replace(/<pre\b[^>]*>([\s\S]*?)<\/pre>/gi, (_match, preContent: string) => {
@@ -137,18 +163,10 @@ function renderHtmlFragmentToMarkdown(html: string, baseUrl: URL): string {
   });
 
   content = content.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, text: string) => {
-    const clean = decodeHtmlEntities(stripTags(text)).trim();
+    const clean = normalizeWhitespace(decodeHtmlEntities(stripTags(text)));
     return clean ? `\n\n${"#".repeat(Number(level))} ${clean}\n\n` : "";
   });
 
-  content = content.replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (_match, attrs: string, text: string) => {
-    const hrefMatch = attrs.match(/href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i);
-    const rawHref = hrefMatch ? hrefMatch[1] || hrefMatch[2] || hrefMatch[3] || "" : "";
-    const href = toAbsoluteUrl(rawHref, baseUrl);
-    const label = normalizeWhitespace(decodeHtmlEntities(stripTags(text))) || href;
-    return href ? `[${label}](${href})` : label;
-  });
-
   content = content.replace(/<img\b([^>]*)>/gi, (_match, attrs: string) => {
     const altMatch = attrs.match(/alt\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i);
     const srcMatch = attrs.match(/src\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i);
@@ -159,6 +177,18 @@ function renderHtmlFragmentToMarkdown(html: string, baseUrl: URL): string {
     return src ? `![${alt}](${src})` : "";
   });
 
+  content = content.replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (_match, attrs: string, text: string) => {
+    const hrefMatch = attrs.match(/href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i);
+    const rawHref = hrefMatch ? hrefMatch[1] || hrefMatch[2] || hrefMatch[3] || "" : "";
+    const href = toAbsoluteUrl(rawHref, baseUrl);
+    const label = normalizeWhitespace(decodeHtmlEntities(stripTags(text)));
+    if (!href || !label) {
+      return label;
+    }
+
+    return `[${label}](${href})`;
+  });
+
   content = content
     .replace(/<(strong|b)\b[^>]*>([\s\S]*?)<\/\1>/gi, (_m, _tag: string, text: string) => {
       const clean = normalizeWhitespace(decodeHtmlEntities(stripTags(text)));
@@ -185,7 +215,7 @@ function renderHtmlFragmentToMarkdown(html: string, baseUrl: URL): string {
   content = decodeHtmlEntities(stripTags(content))
     .replace(/\n{3,}/g, "\n\n")
     .split("\n")
-    .map((line) => line.trimEnd())
+    .map((line) => line.trim())
     .join("\n")
     .trim();
 
@@ -244,8 +274,12 @@ function renderNode(node: any, baseUrl: URL, listDepth: number): string {
   }
 
   if (tag === "a") {
-    const text = renderInlineChildren(el, baseUrl) || normalizeWhitespace(el.getAttribute("href") || "");
+    const text = renderInlineChildren(el, baseUrl);
     const href = toAbsoluteUrl(el.getAttribute("href"), baseUrl);
+    if (!text) {
+      return "";
+    }
+
     return href ? `[${text}](${href})` : text;
   }
 
@@ -349,9 +383,20 @@ function renderInlineChildren(parent: any, baseUrl: URL): string {
     const tag = el.tagName.toLowerCase();
 
     if (tag === "a") {
-      const text = renderInlineChildren(el, baseUrl) || normalizeWhitespace(el.getAttribute("href") || "");
+      const text = renderInlineChildren(el, baseUrl);
       const href = toAbsoluteUrl(el.getAttribute("href"), baseUrl);
-      out.push(href ? `[${text}](${href})` : text);
+      if (text) {
+        out.push(href ? `[${text}](${href})` : text);
+      }
+      continue;
+    }
+
+    if (tag === "img") {
+      const alt = normalizeWhitespace(el.getAttribute("alt") || "image");
+      const src = toAbsoluteUrl(el.getAttribute("src"), baseUrl);
+      if (src) {
+        out.push(`![${alt}](${src})`);
+      }
       continue;
     }
 
@@ -408,3 +453,70 @@ function toAbsoluteUrl(href: string | null, baseUrl: URL): string {
 function normalizeWhitespace(value: string): string {
   return value.replace(/\s+/g, " ").trim();
 }
+
+function pruneDocument(doc: { querySelectorAll: (selector: string) => any }): void {
+  for (const selector of NOISE_SELECTORS) {
+    doc.querySelectorAll(selector).forEach((node: any) => node.remove());
+  }
+}
+
+function selectContentRoot(doc: { querySelector: (selector: string) => any; body?: any }): any {
+  const main = doc.querySelector("main");
+  if (main && /\bhome-page\b/.test(main.getAttribute?.("class") || "")) {
+    return main;
+  }
+
+  const preferredSelectors = [
+    "#main article",
+    "main article",
+    "article.guide",
+    "article",
+    "#main",
+    "[role='main']",
+    "main",
+    "body",
+  ];
+
+  for (const selector of preferredSelectors) {
+    const found = doc.querySelector(selector);
+    if (found) {
+      return found;
+    }
+  }
+
+  return doc.body || null;
+}
+
+function extractPrimaryHtmlFragment(html: string): string {
+  const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
+  const mainHtml = mainMatch ? mainMatch[1] : "";
+  const isHome = /<main\b[^>]*class=("|')[^"']*\bhome-page\b[^"']*\1/i.test(html);
+
+  if (isHome && mainHtml) {
+    return mainHtml;
+  }
+
+  const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+  if (articleMatch) {
+    return articleMatch[1];
+  }
+
+  if (mainHtml) {
+    return mainHtml;
+  }
+
+  return html;
+}
+
+function stripFallbackNoise(html: string): string {
+  return html
+    .replace(/<header\b[^>]*>[\s\S]*?<\/header>/gi, "")
+    .replace(/<footer\b[^>]*>[\s\S]*?<\/footer>/gi, "")
+    .replace(/<nav\b[^>]*>[\s\S]*?<\/nav>/gi, "")
+    .replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "")
+    .replace(/<div\b[^>]*class=("|')[^"']*\bnext-previous\b[^"']*\1[^>]*>[\s\S]*?<\/div>/gi, "")
+    .replace(/<section\b[^>]*class=("|')[^"']*\bhome-banner-info\b[^"']*\1[^>]*>[\s\S]*?<\/section>/gi, "")
+    .replace(/<div\b[^>]*class=("|')[^"']*\bside-bar\b[^"']*\1[^>]*>[\s\S]*?<\/div>/gi, "")
+    .replace(/<a\b[^>]*class=("|')[^"']*\bskip-to-content-link\b[^"']*\1[^>]*>[\s\S]*?<\/a>/gi, "")
+    .replace(/<a\b[^>]*class=("|')[^"']*\bheader-anchor\b[^"']*\1[^>]*>[\s\S]*?<\/a>/gi, "");
+}