|
| 1 | +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; |
| 2 | +import { dirname, join, posix } from 'node:path'; |
| 3 | + |
| 4 | +import type { DefaultTheme } from 'vitepress'; |
| 5 | + |
| 6 | +import { guideSidebar } from './nav'; |
| 7 | + |
| 8 | +/** |
| 9 | + * LLM-facing renditions of the docs, generated into the site output at build |
| 10 | + * time (wired into `buildEnd` in config.mts), following https://llmstxt.org |
| 11 | + * and the Python SDK's equivalent: |
| 12 | + * |
| 13 | + * - `<page>.md` next to every guide page's HTML — the page as plain markdown, |
| 14 | + * frontmatter stripped, links absolutized (cross-page links point at the |
| 15 | + * `.md` renditions so an agent can keep fetching markdown). |
| 16 | + * - `llms.txt` — the index: one line per page, grouped like the sidebar. |
| 17 | + * - `llms-full.txt` — every guide page concatenated in sidebar order. |
| 18 | + * |
| 19 | + * The generated API reference stays out of all three (linked, not inlined): |
| 20 | + * it would swamp the prose that answers most questions. |
| 21 | + */ |
| 22 | + |
| 23 | +interface Page { |
| 24 | + /** Sidebar title. */ |
| 25 | + title: string; |
| 26 | + /** Path of the markdown source relative to the docs dir, e.g. `servers/tools.md`. */ |
| 27 | + sourcePath: string; |
| 28 | + /** Absolute URL of the markdown rendition. */ |
| 29 | + mdUrl: string; |
| 30 | +} |
| 31 | + |
| 32 | +interface Section { |
| 33 | + title: string; |
| 34 | + pages: Page[]; |
| 35 | +} |
| 36 | + |
| 37 | +function toPage(site: string, title: string, route: string): Page { |
| 38 | + const sourcePath = route === '/' ? 'index.md' : route.endsWith('/') ? `${route.slice(1)}index.md` : `${route.slice(1)}.md`; |
| 39 | + return { title, sourcePath, mdUrl: `${site}/${sourcePath}` }; |
| 40 | +} |
| 41 | + |
| 42 | +/** Flatten the sidebar into ordered sections, preserving sidebar order exactly. */ |
| 43 | +function sections(site: string): Section[] { |
| 44 | + const out: Section[] = [{ title: 'Overview', pages: [toPage(site, 'MCP TypeScript SDK', '/')] }]; |
| 45 | + for (const entry of guideSidebar as (DefaultTheme.SidebarItem & { items?: DefaultTheme.SidebarItem[] })[]) { |
| 46 | + if (entry.items) { |
| 47 | + out.push({ title: entry.text!, pages: entry.items.map(i => toPage(site, i.text!, i.link!)) }); |
| 48 | + } else { |
| 49 | + out.push({ title: entry.text!, pages: [toPage(site, entry.text!, entry.link!)] }); |
| 50 | + } |
| 51 | + } |
| 52 | + return out; |
| 53 | +} |
| 54 | + |
| 55 | +/** Read a `description:` frontmatter value, if present. */ |
| 56 | +function frontmatterDescription(markdown: string): string | undefined { |
| 57 | + if (!markdown.startsWith('---\n')) return undefined; |
| 58 | + const end = markdown.indexOf('\n---\n', 4); |
| 59 | + if (end === -1) return undefined; |
| 60 | + const match = /^description:\s*(.+)$/m.exec(markdown.slice(4, end)); |
| 61 | + return match?.[1].trim().replace(/^(['"])(.*)\1$/, '$2'); |
| 62 | +} |
| 63 | + |
| 64 | +/** Strip `--- ... ---` frontmatter. */ |
| 65 | +function stripFrontmatter(markdown: string): string { |
| 66 | + if (!markdown.startsWith('---\n')) return markdown; |
| 67 | + const end = markdown.indexOf('\n---\n', 4); |
| 68 | + return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\n+/, ''); |
| 69 | +} |
| 70 | + |
| 71 | +/** Drop the `source="…"` wiring attribute from fence info lines. */ |
| 72 | +function stripFenceAttributes(markdown: string): string { |
| 73 | + return markdown.replace(/^```(\w+)\s+source="[^"]*"\s*$/gm, '```$1'); |
| 74 | +} |
| 75 | + |
| 76 | +/** |
| 77 | + * Absolutize every link for a page living at `pageDir` (docs-relative, '' for |
| 78 | + * the root). Relative `.md` links stay `.md` so they resolve to the markdown |
| 79 | + * renditions; `/specification/…` goes to modelcontextprotocol.io, mirroring |
| 80 | + * the render-time rewrite in config.mts. |
| 81 | + */ |
| 82 | +function absolutizeLinks(markdown: string, pageDir: string, site: string, sourcePath: string): string { |
| 83 | + // Runs over the raw markdown, code fences included — fine while no code |
| 84 | + // sample contains a relative markdown link (none do; a false rewrite would |
| 85 | + // show up in review of the rendition). |
| 86 | + return markdown.replace(/\]\(([^)\s]+)\)/g, (match, target: string) => { |
| 87 | + if (/^(https?:|mailto:|#)/.test(target)) return match; |
| 88 | + if (target.startsWith('/specification/')) return `](https://modelcontextprotocol.io${target})`; |
| 89 | + if (target.startsWith('/')) return `](${site}${target})`; |
| 90 | + const hashIndex = target.indexOf('#'); |
| 91 | + const path = hashIndex === -1 ? target : target.slice(0, hashIndex); |
| 92 | + const hash = hashIndex === -1 ? '' : target.slice(hashIndex); |
| 93 | + let resolved = posix.normalize(posix.join(pageDir, path)); |
| 94 | + if (resolved.startsWith('..')) { |
| 95 | + throw new Error(`${sourcePath}: relative link escapes the docs root: ${target} — use the site URL or a GitHub URL`); |
| 96 | + } |
| 97 | + if (resolved.startsWith('api/')) { |
| 98 | + // The generated API reference has no markdown renditions — link its HTML. |
| 99 | + resolved = resolved.replace(/\.md$/, '.html'); |
| 100 | + } |
| 101 | + return `](${site}/${resolved}${hash})`; |
| 102 | + }); |
| 103 | +} |
| 104 | + |
| 105 | +function renderPage(docsDir: string, page: Page, site: string): { markdown: string; description?: string } { |
| 106 | + const raw = readFileSync(join(docsDir, page.sourcePath), 'utf8'); |
| 107 | + const pageDir = posix.dirname(page.sourcePath); |
| 108 | + return { |
| 109 | + markdown: absolutizeLinks(stripFenceAttributes(stripFrontmatter(raw)), pageDir === '.' ? '' : pageDir, site, page.sourcePath), |
| 110 | + description: frontmatterDescription(raw) |
| 111 | + }; |
| 112 | +} |
| 113 | + |
| 114 | +/** First prose sentence after the H1, markdown stripped, for the llms.txt index line. */ |
| 115 | +function describe(rendered: string): string { |
| 116 | + const lines = rendered.split('\n'); |
| 117 | + let inFence = false; |
| 118 | + let inContainer = false; |
| 119 | + let past = false; |
| 120 | + const paragraph: string[] = []; |
| 121 | + for (const line of lines) { |
| 122 | + if (line.startsWith('```')) { |
| 123 | + inFence = !inFence; |
| 124 | + continue; |
| 125 | + } |
| 126 | + if (inFence) continue; |
| 127 | + if (line.startsWith(':::')) { |
| 128 | + inContainer = line.trim() !== ':::'; |
| 129 | + continue; |
| 130 | + } |
| 131 | + if (inContainer) continue; |
| 132 | + if (line.startsWith('# ')) { |
| 133 | + past = true; |
| 134 | + continue; |
| 135 | + } |
| 136 | + if (!past) continue; |
| 137 | + if (line.trim() === '' || /^[#>|]|^[-*] /.test(line)) { |
| 138 | + if (paragraph.length > 0) break; |
| 139 | + continue; |
| 140 | + } |
| 141 | + paragraph.push(line.trim()); |
| 142 | + } |
| 143 | + const plain = paragraph |
| 144 | + .join(' ') |
| 145 | + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') |
| 146 | + .replace(/[*`]/g, '') |
| 147 | + .trim(); |
| 148 | + const sentence = /^(.*?[.!?])(\s|$)/.exec(plain); |
| 149 | + const blurb = sentence ? sentence[1] : plain; |
| 150 | + return blurb.length > 220 ? `${blurb.slice(0, 217).replace(/\s+\S*$/, '')}…` : blurb; |
| 151 | +} |
| 152 | + |
| 153 | +/** |
| 154 | + * Generate llms.txt, llms-full.txt, and the per-page markdown renditions into |
| 155 | + * `outDir`. `site` is the absolute site URL without a trailing slash |
| 156 | + * (origin + base), owned by config.mts. |
| 157 | + */ |
| 158 | +export function generateLlmsArtifacts(docsDir: string, outDir: string, site: string): void { |
| 159 | + const groups = sections(site); |
| 160 | + const rendered = new Map<string, { markdown: string; blurb: string }>(); |
| 161 | + |
| 162 | + for (const section of groups) { |
| 163 | + for (const page of section.pages) { |
| 164 | + const { markdown, description } = renderPage(docsDir, page, site); |
| 165 | + rendered.set(page.sourcePath, { markdown, blurb: description ?? describe(markdown) }); |
| 166 | + const outPath = join(outDir, page.sourcePath); |
| 167 | + mkdirSync(dirname(outPath), { recursive: true }); |
| 168 | + writeFileSync(outPath, markdown); |
| 169 | + } |
| 170 | + } |
| 171 | + |
| 172 | + const header = [ |
| 173 | + '# MCP TypeScript SDK', |
| 174 | + '', |
| 175 | + '> The official TypeScript SDK for the Model Context Protocol (MCP): build MCP servers and clients on Node.js, Bun, Deno, and Workers. This is the v2 beta documentation, tracking the 2026-07-28 spec revision.', |
| 176 | + '', |
| 177 | + 'Every page below is also served as plain markdown at its `.md` URL — fetch any page directly.', |
| 178 | + '' |
| 179 | + ]; |
| 180 | + |
| 181 | + const index = [...header]; |
| 182 | + for (const section of groups) { |
| 183 | + index.push(`## ${section.title}`, ''); |
| 184 | + for (const page of section.pages) { |
| 185 | + const blurb = rendered.get(page.sourcePath)!.blurb; |
| 186 | + index.push(`- [${page.title}](${page.mdUrl})${blurb ? `: ${blurb}` : ''}`); |
| 187 | + } |
| 188 | + index.push(''); |
| 189 | + } |
| 190 | + index.push( |
| 191 | + '## Optional', |
| 192 | + '', |
| 193 | + `- [llms-full.txt](${site}/llms-full.txt): every page above concatenated into one file`, |
| 194 | + `- [API reference](${site}/api/): generated per-package API reference`, |
| 195 | + '- [MCP specification](https://modelcontextprotocol.io/specification/latest)', |
| 196 | + '- [v1 documentation](https://ts.sdk.modelcontextprotocol.io/)', |
| 197 | + '' |
| 198 | + ); |
| 199 | + writeFileSync(join(outDir, 'llms.txt'), index.join('\n')); |
| 200 | + |
| 201 | + // A separator no page body contains ('---' appears in the guides as a |
| 202 | + // thematic break, so it cannot delimit pages unambiguously). |
| 203 | + const separator = '='.repeat(80); |
| 204 | + const full = [...header]; |
| 205 | + for (const section of groups) { |
| 206 | + for (const page of section.pages) { |
| 207 | + full.push(separator, `Source: ${page.mdUrl}`, separator, '', rendered.get(page.sourcePath)!.markdown.trimEnd(), ''); |
| 208 | + } |
| 209 | + } |
| 210 | + writeFileSync(join(outDir, 'llms-full.txt'), full.join('\n')); |
| 211 | + |
| 212 | + console.log(`[docs] llms.txt, llms-full.txt, and ${rendered.size} markdown renditions written`); |
| 213 | +} |
0 commit comments