Plugin Docs Parser: Reduce dependency footprint (#2659)

Author: sunker

package-lock.json

@@ -45,11 +45,8 @@
   },
   "dependencies": {
     "@types/hast": "^3.0.0",
-    "gray-matter": "^4.0.3",
-    "hast-util-to-string": "^3.0.0",
-    "rehype-raw": "^7.0.0",
+    "js-yaml": "^4.1.1",
     "rehype-sanitize": "^6.0.0",
-    "rehype-slug": "^6.0.0",
     "remark-gfm": "^4.0.0",
     "remark-parse": "^11.0.0",
     "remark-rehype": "^11.0.0",
@@ -58,6 +55,7 @@
     "vfile": "^6.0.0"
   },
   "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
     "hast-util-to-html": "^9.0.0"
   }
 }

packages/plugin-docs-parser/package.json

@@ -86,7 +86,7 @@ Body text here.`;
     expect(html).toContain('Title');
   });
 
-  it('should preserve safe raw HTML elements like <details>', () => {
+  it('should drop raw HTML elements per the docs spec', () => {
     const markdown = `## FAQ
 
 <details>
@@ -99,8 +99,10 @@ It works by parsing markdown.
     const result = parseMarkdown(markdown);
     const html = toHtml(result.hast);
 
-    expect(html).toContain('<details>');
-    expect(html).toContain('<summary>');
+    // raw HTML is forbidden by the docs spec, so <details>/<summary> get stripped
+    expect(html).not.toContain('<details>');
+    expect(html).not.toContain('<summary>');
+    // text content inside the raw block is preserved as a regular paragraph
     expect(html).toContain('It works by parsing markdown.');
   });
 

packages/plugin-docs-parser/src/parser.test.ts

@@ -2,11 +2,10 @@ import { unified } from 'unified';
 import remarkParse from 'remark-parse';
 import remarkGfm from 'remark-gfm';
 import remarkRehype from 'remark-rehype';
-import rehypeRaw from 'rehype-raw';
-import rehypeSlug from 'rehype-slug';
 import rehypeSanitize, { defaultSchema } from 'rehype-sanitize';
-import matter from 'gray-matter';
+import * as yaml from 'js-yaml';
 import { VFile } from 'vfile';
+import { rehypeSlug } from './plugins/rehype-slug.js';
 import type { Root as HastRoot } from 'hast';
 import { rehypeRewriteAssetPaths } from './plugins/rehype-rewrite-asset-paths.js';
 import { rehypeRewriteDocLinks } from './plugins/rehype-rewrite-doc-links.js';
@@ -63,6 +62,21 @@ export interface ParsedMarkdown {
 // the default schema already allows id (via clobber) and className on code (for language-* classes).
 const sanitizeSchema = { ...defaultSchema, clobberPrefix: '' };
 
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/;
+
+function parseFrontmatter(content: string): { data: Record<string, unknown>; content: string } {
+  const match = content.match(FRONTMATTER_RE);
+  if (!match) {
+    return { data: {}, content };
+  }
+  // yaml.load can return scalars or arrays for malformed frontmatter; coerce
+  // anything that isn't a plain object back to {} so the public type holds.
+  const parsed = yaml.load(match[1]);
+  const data =
+    parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
+  return { data, content: match[2] };
+}
+
 /**
  * Parses markdown content into a HAST tree with extracted frontmatter and headings.
  *
@@ -72,30 +86,23 @@ const sanitizeSchema = { ...defaultSchema, clobberPrefix: '' };
  * @throws {Error} If markdown parsing fails
  */
 export function parseMarkdown(content: string, options?: ParseOptions): ParsedMarkdown {
-  // extract frontmatter using gray-matter
+  // extract frontmatter
   let frontmatter: Record<string, unknown>;
   let markdownContent: string;
 
   try {
-    const result = matter(content);
+    const result = parseFrontmatter(content);
     frontmatter = result.data;
     markdownContent = result.content;
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     throw new Error(`Failed to extract frontmatter: ${message}`);
   }
 
-  // build the unified pipeline: markdown → mdast → hast
-  const processor = unified()
-    .use(remarkParse)
-    .use(remarkGfm)
-    // allow raw HTML in markdown (e.g. <details>, <img>) to pass through to hast;
-    // rehype-raw parses the raw nodes into proper hast elements so
-    // rehype-sanitize can inspect and strip anything dangerous
-    .use(remarkRehype, { allowDangerousHtml: true })
-    .use(rehypeRaw)
-    .use(rehypeStripH1)
-    .use(rehypeSlug);
+  // build the unified pipeline: markdown → mdast → hast.
+  // raw HTML in markdown is forbidden by the docs spec, so allowDangerousHtml
+  // is off and any inline HTML gets dropped by remark-rehype.
+  const processor = unified().use(remarkParse).use(remarkGfm).use(remarkRehype).use(rehypeStripH1).use(rehypeSlug);
 
   // rewrite asset paths before sanitization so URLs are final
   if (options?.assetBaseUrl) {

packages/plugin-docs-parser/src/parser.ts

@@ -1,9 +1,19 @@
 import type { Root, Element } from 'hast';
+import type { Node } from 'unist';
 import type { VFile } from 'vfile';
 import { visit } from 'unist-util-visit';
-import { toString } from 'hast-util-to-string';
 import type { Heading } from '../types.js';
 
+function hastToString(node: Node): string {
+  if ('value' in node) {
+    return node.value as string;
+  }
+  if ('children' in node) {
+    return (node.children as Node[]).map(hastToString).join('');
+  }
+  return '';
+}
+
 declare module 'vfile' {
   interface DataMap {
     headings: Heading[];
@@ -31,7 +41,7 @@ export function rehypeExtractHeadings() {
       headings.push({
         level: node.tagName === 'h2' ? 2 : 3,
         id,
-        text: toString(node),
+        text: hastToString(node),
       });
     });
 

packages/plugin-docs-parser/src/plugins/rehype-extract-headings.ts

@@ -0,0 +1,115 @@
+import { describe, it, expect } from 'vitest';
+import type { Root, Element, Text, Properties } from 'hast';
+import { rehypeSlug } from './rehype-slug.js';
+
+function el(tag: string, text: string, properties: Properties = {}): Element {
+  const child: Text = { type: 'text', value: text };
+  return { type: 'element', tagName: tag, properties, children: [child] };
+}
+
+function tree(...children: Element[]): Root {
+  return { type: 'root', children };
+}
+
+function getEl(root: Root, index: number): Element {
+  const child = root.children[index];
+  if (child.type !== 'element') {
+    throw new Error(`expected element at index ${index}, got ${child.type}`);
+  }
+  return child;
+}
+
+describe('rehypeSlug', () => {
+  const transform = rehypeSlug();
+
+  it('should add id attributes to headings', () => {
+    const root = tree(el('h2', 'Getting Started'), el('h3', 'Prerequisites'));
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'getting-started' });
+    expect(getEl(root, 1).properties).toMatchObject({ id: 'prerequisites' });
+  });
+
+  it('should lowercase and strip punctuation when slugifying', () => {
+    const root = tree(el('h2', "What's New?"));
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'whats-new' });
+  });
+
+  it('should suffix duplicate headings with -1, -2, ...', () => {
+    const root = tree(el('h2', 'Setup'), el('h2', 'Setup'), el('h2', 'Setup'));
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'setup' });
+    expect(getEl(root, 1).properties).toMatchObject({ id: 'setup-1' });
+    expect(getEl(root, 2).properties).toMatchObject({ id: 'setup-2' });
+  });
+
+  it('should leave pre-existing ids untouched', () => {
+    const root = tree(el('h2', 'Setup', { id: 'custom-anchor' }));
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'custom-anchor' });
+  });
+
+  it('should not count pre-existing ids toward dedupe', () => {
+    const root = tree(el('h2', 'Setup', { id: 'setup' }), el('h2', 'Setup'));
+    transform(root);
+
+    // the second heading still gets the unsuffixed slug because the visitor
+    // only tracks ids it has assigned itself
+    expect(getEl(root, 1).properties).toMatchObject({ id: 'setup' });
+  });
+
+  it('should skip headings that slugify to empty (emoji or punctuation only)', () => {
+    const root = tree(el('h2', '🎉'), el('h2', '!!!'));
+    transform(root);
+
+    expect(getEl(root, 0).properties).not.toHaveProperty('id');
+    expect(getEl(root, 1).properties).not.toHaveProperty('id');
+  });
+
+  it('should slugify across all h1-h6 levels', () => {
+    const root = tree(el('h1', 'Title'), el('h4', 'Deep'), el('h6', 'Deeper'));
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'title' });
+    expect(getEl(root, 1).properties).toMatchObject({ id: 'deep' });
+    expect(getEl(root, 2).properties).toMatchObject({ id: 'deeper' });
+  });
+
+  it('should ignore non-heading elements', () => {
+    const root = tree(el('p', 'A paragraph'), el('div', 'A div'));
+    transform(root);
+
+    expect(getEl(root, 0).properties).not.toHaveProperty('id');
+    expect(getEl(root, 1).properties).not.toHaveProperty('id');
+  });
+
+  it('should concatenate text from nested children', () => {
+    const inner: Element = {
+      type: 'element',
+      tagName: 'code',
+      properties: {},
+      children: [{ type: 'text', value: 'parseMarkdown' } as Text],
+    };
+    const heading: Element = {
+      type: 'element',
+      tagName: 'h2',
+      properties: {},
+      children: [{ type: 'text', value: 'The ' } as Text, inner, { type: 'text', value: ' function' } as Text],
+    };
+    const root = tree(heading);
+    transform(root);
+
+    expect(getEl(root, 0).properties).toMatchObject({ id: 'the-parsemarkdown-function' });
+  });
+
+  it('should handle an empty tree', () => {
+    const root = tree();
+    transform(root);
+
+    expect(root.children).toHaveLength(0);
+  });
+});