|
| 1 | +// Generates website/static/llms-full.txt — the full text of the canonical docs |
| 2 | +// concatenated into a single file, served verbatim at /llms-full.txt (the |
| 3 | +// "give me everything" companion to the hand-curated /llms.txt index). |
| 4 | +// |
| 5 | +// Source of truth is repo-root docs/, mirroring the docs plugin's include rule |
| 6 | +// in docusaurus.config.ts (intro.md, adopters/**, contributors/**). The output |
| 7 | +// is a generated artifact (gitignored) and is regenerated by prestart/prebuild, |
| 8 | +// exactly like scripts/sync-docs-i18n.ts. |
| 9 | +import {readdirSync, readFileSync, writeFileSync, statSync} from 'node:fs'; |
| 10 | +import {join, resolve} from 'node:path'; |
| 11 | +import matter from 'gray-matter'; |
| 12 | + |
| 13 | +const ROOT = resolve(__dirname, '..'); |
| 14 | +const DOCS = resolve(ROOT, '../docs'); |
| 15 | +const OUT = join(ROOT, 'static', 'llms-full.txt'); |
| 16 | +const SITE_URL = 'https://straymark.dev'; |
| 17 | + |
| 18 | +// Directories under docs/ to walk, in output order. intro.md is handled first |
| 19 | +// and separately (it carries `slug: /`). decisions/ and proposals/ are excluded |
| 20 | +// by omission — they live outside these two dirs. |
| 21 | +const DIRS = ['adopters', 'contributors']; |
| 22 | + |
| 23 | +/** Recursively collect *.md files under `dir`, returned sorted for stable output. */ |
| 24 | +function collectMarkdown(dir: string): string[] { |
| 25 | + const out: string[] = []; |
| 26 | + for (const entry of readdirSync(dir)) { |
| 27 | + const full = join(dir, entry); |
| 28 | + if (statSync(full).isDirectory()) { |
| 29 | + out.push(...collectMarkdown(full)); |
| 30 | + } else if (entry.endsWith('.md')) { |
| 31 | + out.push(full); |
| 32 | + } |
| 33 | + } |
| 34 | + return out.sort(); |
| 35 | +} |
| 36 | + |
| 37 | +/** Map an absolute doc path to its public Docusaurus route (trailingSlash: false). */ |
| 38 | +function routeFor(absPath: string): string { |
| 39 | + const rel = absPath.slice(DOCS.length + 1).replace(/\.md$/, ''); |
| 40 | + if (rel === 'intro') return `${SITE_URL}/docs`; // slug: / → docs root |
| 41 | + // README.md / index.md become the directory index route. |
| 42 | + const asIndex = rel.replace(/\/(README|index)$/, ''); |
| 43 | + if (asIndex !== rel) return `${SITE_URL}/docs/${asIndex}`; |
| 44 | + return `${SITE_URL}/docs/${rel}`; |
| 45 | +} |
| 46 | + |
| 47 | +/** Title from frontmatter `title`, else first `# H1`, else the filename. */ |
| 48 | +function titleFor(absPath: string, data: Record<string, unknown>, body: string): string { |
| 49 | + if (typeof data.title === 'string' && data.title.trim()) return data.title.trim(); |
| 50 | + const h1 = body.match(/^#\s+(.+)$/m); |
| 51 | + if (h1) return h1[1].trim(); |
| 52 | + return absPath.slice(DOCS.length + 1).replace(/\.md$/, ''); |
| 53 | +} |
| 54 | + |
| 55 | +// Ordered file list: intro first, then each configured dir. |
| 56 | +const files = [join(DOCS, 'intro.md'), ...DIRS.flatMap((d) => collectMarkdown(join(DOCS, d)))]; |
| 57 | + |
| 58 | +const sections: string[] = [ |
| 59 | + '# StrayMark — Full Documentation', |
| 60 | + '', |
| 61 | + '> The complete text of the StrayMark adopter and contributor documentation,', |
| 62 | + '> concatenated for LLM consumption. The curated index lives at /llms.txt.', |
| 63 | + '> Canonical source: https://github.com/StrangeDaysTech/straymark/tree/main/docs', |
| 64 | + '', |
| 65 | +]; |
| 66 | + |
| 67 | +for (const file of files) { |
| 68 | + const raw = readFileSync(file, 'utf8'); |
| 69 | + const {content, data} = matter(raw); |
| 70 | + const title = titleFor(file, data, content); |
| 71 | + const url = routeFor(file); |
| 72 | + sections.push('---', '', `## ${title}`, '', `Source: ${url}`, '', content.trim(), ''); |
| 73 | +} |
| 74 | + |
| 75 | +writeFileSync(OUT, sections.join('\n') + '\n'); |
| 76 | +console.log(`[gen-llms-full] wrote ${OUT} (${files.length} docs)`); |
0 commit comments