fix(md): <main> content stripping

Author: tmm

src/md/fromHtml.test.ts

@@ -616,6 +616,34 @@ describe('strips form elements', () => {
     expect(result).not.toContain('Page 0')
   })
 
+  test('preserves high link density content inside main', async () => {
+    const links = Array.from(
+      { length: 10 },
+      (_, i) => `<a href="/news/${i}">Research update ${i}</a>`,
+    ).join(' ')
+    const { content: result } = await fromHtml(
+      html({
+        body: `<main><h1>News</h1><section>${links}</section></main>`,
+      }),
+    )
+    expect(result).toContain('# News')
+    expect(result).toContain('Research update 0')
+  })
+
+  test('preserves high link density content inside role main', async () => {
+    const links = Array.from(
+      { length: 10 },
+      (_, i) => `<a href="/news/${i}">Product announcement ${i}</a>`,
+    ).join(' ')
+    const { content: result } = await fromHtml(
+      html({
+        body: `<div role="main"><h1>Updates</h1><section>${links}</section></div>`,
+      }),
+    )
+    expect(result).toContain('# Updates')
+    expect(result).toContain('Product announcement 0')
+  })
+
   test('preserves content wrappers that contain an article', async () => {
     const links = Array.from(
       { length: 10 },

src/md/fromHtml.ts

@@ -153,7 +153,7 @@ const linkDensityBlockTags = new Set(['div', 'ol', 'section', 'ul'])
 
 function rehypeStripNoise(profile?: Profile<Record<string, unknown>>) {
   return (tree: Root) => {
-    strip(tree, false, profile)
+    strip(tree, false, false, profile)
   }
 }
 
@@ -162,13 +162,16 @@ const sectioningTags = new Set(['article', 'main', 'section'])
 function strip(
   node: Element | Root,
   inSectioning = false,
+  inContentContainer = false,
   profile?: Profile<Record<string, unknown>>,
 ) {
   if (!node.children) return
   node.children = node.children.filter((child) => {
     if (child.type === 'comment') return false
     if (child.type !== 'element') return true
     const knownContentRoot = isKnownContentRoot(child, profile)
+    const childInContentContainer =
+      inContentContainer || knownContentRoot || isContentContainer(child)
 
     if (strippedTagNames.has(child.tagName)) return false
 
@@ -183,17 +186,24 @@ function strip(
     if (isSkipLink(child)) return false
     if (!knownContentRoot && !containsContentContainer(child) && matchesNoiseClassId(child))
       return false
-    if (!knownContentRoot && isHighLinkDensity(child)) return false
+    if (!childInContentContainer && isHighLinkDensity(child)) return false
 
-    strip(child, inSectioning || sectioningTags.has(child.tagName), profile)
+    strip(
+      child,
+      inSectioning || sectioningTags.has(child.tagName),
+      childInContentContainer,
+      profile,
+    )
     return true
   })
 }
 
 function containsContentContainer(node: Element): boolean {
-  return (
-    node.tagName === 'article' || node.tagName === 'main' || hasDescendantContentContainer(node)
-  )
+  return isContentContainer(node) || hasDescendantContentContainer(node)
+}
+
+function isContentContainer(node: Element): boolean {
+  return node.tagName === 'article' || node.tagName === 'main' || node.properties?.role === 'main'
 }
 
 function isKnownContentRoot(node: Element, profile?: Profile<Record<string, unknown>>): boolean {
@@ -297,7 +307,7 @@ function getLinkTextLength(node: Element): number {
 function hasDescendantContentContainer(node: Element): boolean {
   for (const child of node.children) {
     if (child.type !== 'element') continue
-    if (child.tagName === 'article' || child.tagName === 'main') return true
+    if (isContentContainer(child)) return true
     if (hasDescendantContentContainer(child)) return true
   }
   return false