diff --git a/.gitignore b/.gitignore index ceb713d..adf1a8e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ dist/ # generated types .astro/ public/snapshot +public/llms.txt +public/docs-index.json # dependencies node_modules/ diff --git a/astro.config.mjs b/astro.config.mjs index c2ad9fc..a02cb24 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -3,6 +3,7 @@ import { defineConfig } from "astro/config"; import starlight from "@astrojs/starlight"; import ecTwoSlash from "expressive-code-twoslash"; import topics from "starlight-sidebar-topics"; +import starlightMarkdown from "starlight-markdown"; const site = "https://bomb.sh/docs/"; @@ -87,6 +88,7 @@ export default defineConfig({ { icon: 'github', label: 'GitHub', href: 'https://bomb.sh/on/github' }, ], plugins: [ + starlightMarkdown(), topics([ { label: "Clack", diff --git a/package.json b/package.json index 42ff298..eb0d409 100644 --- a/package.json +++ b/package.json @@ -5,11 +5,13 @@ "scripts": { "dev": "astro dev", "start": "astro dev", - "prebuild": "pnpm run snapshot", + "prebuild": "pnpm run snapshot && pnpm run generate:docs-index", + "predev": "pnpm run generate:docs-index", "build": "astro build && cp public/_headers dist/_headers", "preview": "astro preview", "astro": "astro", - "snapshot": "node --experimental-strip-types ./scripts/snapshot.ts" + "snapshot": "node --experimental-strip-types ./scripts/snapshot.ts", + "generate:docs-index": "node --experimental-strip-types ./scripts/generate-docs-index.ts" }, "dependencies": { "@astrojs/starlight": "^0.37.1", @@ -26,6 +28,7 @@ "astro": "^5.16.6", "expressive-code-twoslash": "^0.5.3", "sharp": "^0.33.5", + "starlight-markdown": "^0.1.5", "starlight-sidebar-topics": "^0.6.2" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b139714..8b68dd5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: sharp: specifier: ^0.33.5 version: 0.33.5 + starlight-markdown: + specifier: ^0.1.5 + version: 0.1.5(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)) starlight-sidebar-topics: specifier: ^0.6.2 version: 0.6.2(@astrojs/starlight@0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2))) @@ -2139,6 +2142,11 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + starlight-markdown@0.1.5: + resolution: {integrity: sha512-23LXRaZp7pyE+r/HP6rxHfwic8HfvUBT4EImECA6encs/eTtrF0Z+7svANofdtfbiNt31D5q26i03B6FtcSmGg==} + peerDependencies: + astro: ^5.0.0 + starlight-sidebar-topics@0.6.2: resolution: {integrity: sha512-SNCTUZS/hcVor0ZcaXbaSVU37+V+qtvzNirkvnOg3Mqu/awuGpthkH5+uKpiZqWxLffp6TrOlsv5E5QsxrndNg==} engines: {node: '>=18'} @@ -5035,6 +5043,10 @@ snapshots: space-separated-tokens@2.0.2: {} + starlight-markdown@0.1.5(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)): + dependencies: + astro: 5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2) + starlight-sidebar-topics@0.6.2(@astrojs/starlight@0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2))): dependencies: '@astrojs/starlight': 0.37.1(astro@5.16.6(@types/node@22.19.3)(rollup@4.55.1)(typescript@5.8.2)) diff --git a/public/_headers b/public/_headers index 23edcb1..230419b 100644 --- a/public/_headers +++ b/public/_headers @@ -3,3 +3,4 @@ Cross-Origin-Opener-Policy: same-origin Cross-Origin-Resource-Policy: cross-origin Referrer-Policy: strict-origin-when-cross-origin + Vary: Accept diff --git a/scripts/generate-docs-index.ts b/scripts/generate-docs-index.ts new file mode 100644 index 0000000..0a96d51 --- /dev/null +++ b/scripts/generate-docs-index.ts @@ -0,0 +1,161 @@ +/** + * Walks `src/content/docs` and emits `public/llms.txt` plus + * `public/docs-index.json` for agent discoverability and offline search. + */ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const rootDir = fileURLToPath(new URL('../', import.meta.url)); +const docsDir = path.join(rootDir, 'src/content/docs'); +const BASE_URL = 'https://bomb.sh/docs'; + +interface DocPage { + slug: string; + title: string; + description: string; + url: string; + markdownUrl: string; + template?: string; +} + +function stripQuotes(value: string): string { + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return value.slice(1, -1); + } + return value; +} + +function parseFrontmatter(content: string): Record { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!match) return {}; + + const result: Record = {}; + for (const line of match[1].split('\n')) { + const kv = line.match(/^([\w-]+):\s*(.+)$/); + if (!kv) continue; + result[kv[1]] = stripQuotes(kv[2].trim()); + } + return result; +} + +function filePathToSlug(relativePath: string): string { + const withoutExt = relativePath.replace(/\.mdx?$/, ''); + if (withoutExt === 'index') return ''; + if (withoutExt.endsWith('/index')) { + return withoutExt.slice(0, -'/index'.length); + } + return withoutExt; +} + +function pageUrl(slug: string): string { + return slug ? `${BASE_URL}/${slug}/` : `${BASE_URL}/`; +} + +function markdownUrl(slug: string): string { + return slug ? `${BASE_URL}/${slug}/index.md` : `${BASE_URL}/index.md`; +} + +async function walkDocs(dir: string, base = ''): Promise { + const pages: DocPage[] = []; + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const rel = base ? `${base}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + pages.push(...(await walkDocs(path.join(dir, entry.name), rel))); + continue; + } + + if (!entry.name.endsWith('.mdx') && !entry.name.endsWith('.md')) { + continue; + } + + const content = await fs.readFile(path.join(dir, entry.name), 'utf8'); + const frontmatter = parseFrontmatter(content); + const slug = filePathToSlug(rel); + + pages.push({ + slug, + title: frontmatter.title ?? (slug || 'Untitled'), + description: frontmatter.description ?? '', + url: pageUrl(slug), + markdownUrl: markdownUrl(slug), + template: frontmatter.template, + }); + } + + return pages; +} + +function isIndexed(page: DocPage): boolean { + if (page.slug === '404') return false; + if (page.template === 'splash' && page.slug !== '') return false; + return true; +} + +function generateLlmsTxt(pages: DocPage[]): string { + const indexed = pages.filter(isIndexed); + const lines = [ + '# Bombshell Documentation', + '', + '> Effortlessly build beautiful command-line apps. Docs for Clack, Args, and Tab.', + '', + `Canonical docs: ${BASE_URL}/`, + '', + ]; + + const homepage = indexed.find((page) => page.slug === ''); + if (homepage) { + lines.push(`- [${homepage.title}](${homepage.url}): ${homepage.description}`, ''); + } + + const sections = new Map(); + for (const page of indexed) { + if (page.slug === '') continue; + const section = page.slug.split('/')[0]; + if (!sections.has(section)) sections.set(section, []); + sections.get(section)!.push(page); + } + + for (const [section, sectionPages] of [...sections.entries()].sort()) { + const label = section.charAt(0).toUpperCase() + section.slice(1); + lines.push(`## ${label}`, ''); + for (const page of sectionPages.sort((a, b) => a.slug.localeCompare(b.slug))) { + lines.push(`- [${page.title}](${page.url}): ${page.description}`); + } + lines.push(''); + } + + return `${lines.join('\n').trimEnd()}\n`; +} + +async function main() { + const pages = await walkDocs(docsDir); + const indexed = pages.filter(isIndexed); + + await fs.writeFile( + path.join(rootDir, 'public/docs-index.json'), + `${JSON.stringify( + { + generatedAt: new Date().toISOString(), + baseUrl: BASE_URL, + pages: indexed, + }, + null, + 2, + )}\n`, + ); + await fs.writeFile(path.join(rootDir, 'public/llms.txt'), generateLlmsTxt(pages)); + + console.log(`Generated docs index with ${indexed.length} pages`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +});