diff --git a/scripts/build-docs.test.ts b/scripts/build-docs.test.ts index 471b77016b..f10bdd4199 100644 --- a/scripts/build-docs.test.ts +++ b/scripts/build-docs.test.ts @@ -7511,6 +7511,170 @@ description: Generated API docs - [API Documentation]({{SITE_URL}}/docs/api-doc): Generated API docs`) }) + test('Should group SDK-scoped pages under sub-headers in llms.txt overview', async () => { + const { tempDir, readFile } = await createTempFiles([ + { + path: './docs/manifest.json', + content: JSON.stringify({ + navigation: [ + [ + { title: 'Generic Doc', href: '/docs/generic-doc' }, + { title: 'SDK Doc', href: '/docs/sdk-doc' }, + ], + ], + }), + }, + { + path: './docs/generic-doc.mdx', + content: `--- +title: Generic Doc +description: A generic guide +--- + +# Generic Doc +`, + }, + { + path: './docs/sdk-doc.mdx', + content: `--- +title: SDK Doc +description: An SDK-scoped guide +sdk: nextjs, react +--- + +# SDK Doc +`, + }, + ]) + + await build( + await createConfig({ + ...baseConfig, + basePath: tempDir, + validSdks: ['nextjs', 'react'], + llms: { + overviewPath: 'llms.txt', + }, + }), + ) + + expect(await readFile('./dist/llms.txt')).toEqual(`# Clerk + +## Docs + +- [Generic Doc]({{SITE_URL}}/docs/generic-doc): A generic guide + +### Next.js + +- [SDK Doc]({{SITE_URL}}/docs/nextjs/sdk-doc): An SDK-scoped guide + +### React + +- [SDK Doc]({{SITE_URL}}/docs/react/sdk-doc): An SDK-scoped guide`) + }) + + test('Should group reference// pages and URL-aliased SDKs under their SDK sub-header', async () => { + const { tempDir, readFile } = await createTempFiles([ + { + path: './docs/manifest.json', + content: JSON.stringify({ + navigation: [ + [ + { title: 'Generic Guide', href: '/docs/generic-guide' }, + { title: 'Vue Plugin', href: '/docs/reference/vue/clerk-plugin' }, + { title: 'Astro Middleware', href: '/docs/reference/astro/clerk-middleware' }, + { title: 'JS Overview', href: '/docs/reference/javascript/overview' }, + { title: 'Express Middleware', href: '/docs/reference/express/clerk-middleware' }, + ], + ], + }), + }, + { + path: './docs/generic-guide.mdx', + content: `--- +title: Generic Guide +description: A generic guide +--- + +# Generic Guide +`, + }, + { + path: './docs/reference/vue/clerk-plugin.mdx', + content: `--- +title: clerkPlugin +description: Vue clerkPlugin reference +--- + +# clerkPlugin +`, + }, + { + path: './docs/reference/astro/clerk-middleware.mdx', + content: `--- +title: clerkMiddleware (Astro) +description: Astro middleware reference +--- + +# clerkMiddleware +`, + }, + { + path: './docs/reference/javascript/overview.mdx', + content: `--- +title: JavaScript Overview +description: JS frontend SDK overview +--- + +# JavaScript Overview +`, + }, + { + path: './docs/reference/express/clerk-middleware.mdx', + content: `--- +title: clerkMiddleware (Express) +description: Express middleware reference +--- + +# clerkMiddleware +`, + }, + ]) + + await build( + await createConfig({ + ...baseConfig, + basePath: tempDir, + validSdks: ['vue', 'astro', 'js-frontend', 'expressjs'], + llms: { + overviewPath: 'llms.txt', + }, + }), + ) + + expect(await readFile('./dist/llms.txt')).toEqual(`# Clerk + +## Docs + +- [Generic Guide]({{SITE_URL}}/docs/generic-guide): A generic guide + +### Vue + +- [clerkPlugin]({{SITE_URL}}/docs/reference/vue/clerk-plugin): Vue clerkPlugin reference + +### Astro + +- [clerkMiddleware (Astro)]({{SITE_URL}}/docs/reference/astro/clerk-middleware): Astro middleware reference + +### JavaScript + +- [JavaScript Overview]({{SITE_URL}}/docs/reference/javascript/overview): JS frontend SDK overview + +### Express + +- [clerkMiddleware (Express)]({{SITE_URL}}/docs/reference/express/clerk-middleware): Express middleware reference`) + }) + test('Should output llms-full.txt full pages', async () => { const { tempDir, readFile } = await createTempFiles([ { diff --git a/scripts/build-docs.ts b/scripts/build-docs.ts index c1f05b0e03..0dd0bd0172 100644 --- a/scripts/build-docs.ts +++ b/scripts/build-docs.ts @@ -1212,7 +1212,7 @@ ${yaml.stringify({ } if (config.llms?.overviewPath) { - const llms = await generateLLMs(outputtedDocsFiles) + const llms = await generateLLMs(outputtedDocsFiles, config.validSdks) await writeFile(config.llms.overviewPath, llms) } } diff --git a/scripts/lib/llms.test.ts b/scripts/lib/llms.test.ts index 7cc176dde3..ab9e0bd38a 100644 --- a/scripts/lib/llms.test.ts +++ b/scripts/lib/llms.test.ts @@ -8,6 +8,8 @@ describe('formatLLMsDocLine', () => { title: 'Quickstart', url: 'https://example.com/docs/quickstart', description: 'Get up and running with Clerk in minutes.', + path: 'quickstart.mdx', + content: '', }), ).toBe('- [Quickstart](https://example.com/docs/quickstart): Get up and running with Clerk in minutes.') }) @@ -17,6 +19,9 @@ describe('formatLLMsDocLine', () => { formatLLMsDocLine({ title: 'Quickstart', url: 'https://example.com/docs/quickstart', + description: undefined, + path: 'quickstart.mdx', + content: '', }), ).toBe('- [Quickstart](https://example.com/docs/quickstart)') }) @@ -168,22 +173,25 @@ Body`, describe('writeLLMs', () => { test('renders a markdown list with descriptions when available', async () => { - const result = await writeLLMs([ - { - path: 'quickstart.mdx', - url: '{{SITE_URL}}/docs/quickstart', - content: '', - title: 'Quickstart', - description: 'Get up and running with Clerk in minutes.', - }, - { - path: 'overview.mdx', - url: '{{SITE_URL}}/docs/overview', - content: '', - title: 'Overview', - description: undefined, - }, - ]) + const result = await writeLLMs( + [ + { + path: 'quickstart.mdx', + url: '{{SITE_URL}}/docs/quickstart', + content: '', + title: 'Quickstart', + description: 'Get up and running with Clerk in minutes.', + }, + { + path: 'overview.mdx', + url: '{{SITE_URL}}/docs/overview', + content: '', + title: 'Overview', + description: undefined, + }, + ], + ['react'], + ) expect(result).toBe( [ diff --git a/scripts/lib/llms.ts b/scripts/lib/llms.ts index 71886672d5..c3fba9d12f 100644 --- a/scripts/lib/llms.ts +++ b/scripts/lib/llms.ts @@ -1,18 +1,89 @@ import yaml from 'yaml' +import type { SDK } from './schemas' type Docs = Map +// Display names for SDKs when rendered as sub-headers in llms.txt. +// Keep these in sync with VALID_SDKS in ./schemas.ts. +const SDK_DISPLAY_NAMES: Record = { + nextjs: 'Next.js', + react: 'React', + 'js-frontend': 'JavaScript', + 'chrome-extension': 'Chrome Extension', + expo: 'Expo', + android: 'Android', + ios: 'iOS', + expressjs: 'Express', + fastify: 'Fastify', + 'react-router': 'React Router', + 'tanstack-react-start': 'TanStack React Start', + go: 'Go', + astro: 'Astro', + nuxt: 'Nuxt', + vue: 'Vue', + ruby: 'Ruby', +} + +// Some SDKs are referenced in URL/file paths under a slug that doesn't match +// their SDK key (e.g. /docs/reference/javascript/... is the js-frontend SDK). +// This map allows those path segments to be recognized as SDK-scoped. +const PATH_SEGMENT_SDK_ALIASES: Record = { + javascript: 'js-frontend', + express: 'expressjs', +} + +const getSdkFromPath = (path: string, validSdks: readonly SDK[]): SDK | null => { + // Skip the trailing file segment (e.g. "expo.mdx") - we only want directory + // segments, so a generic doc whose filename contains an SDK name is not + // misclassified as SDK-scoped. + const segments = path.split('/').slice(0, -1) + for (const segment of segments) { + if (validSdks.includes(segment as SDK)) { + return segment as SDK + } + const aliased = PATH_SEGMENT_SDK_ALIASES[segment] + if (aliased && validSdks.includes(aliased)) { + return aliased + } + } + return null +} + +const getSdkDisplayName = (sdk: SDK): string => SDK_DISPLAY_NAMES[sdk] ?? sdk + export const writeLLMsFull = async (outputtedDocsFiles: OutputtedDocsFiles) => { return outputtedDocsFiles.map((file) => file.content).join('\n') } -export const formatLLMsDocLine = (page: { title: string; url: string; description?: string }) => { - return page.description ? `- [${page.title}](${page.url}): ${page.description}` : `- [${page.title}](${page.url})` -} +export const formatLLMsDocLine = (page: OutputtedDocsFiles[number]) => + page.description ? `- [${page.title}](${page.url}): ${page.description}` : `- [${page.title}](${page.url})` + +export const writeLLMs = async (outputtedDocsFiles: OutputtedDocsFiles, validSdks: readonly SDK[]) => { + const generic: OutputtedDocsFiles = [] + const bySdk = new Map() + + for (const page of outputtedDocsFiles) { + const sdk = getSdkFromPath(page.path, validSdks) + if (sdk === null) { + generic.push(page) + } else { + const list = bySdk.get(sdk) ?? [] + list.push(page) + bySdk.set(sdk, list) + } + } + + const sections: string[] = [`## Docs`, generic.map(formatLLMsDocLine).join('\n')] + + // Emit SDK sections in the order they appear in validSdks for stable, predictable output. + for (const sdk of validSdks) { + const pages = bySdk.get(sdk) + if (!pages || pages.length === 0) continue + sections.push(`### ${getSdkDisplayName(sdk)}`) + sections.push(pages.map(formatLLMsDocLine).join('\n')) + } -export const writeLLMs = async (outputtedDocsFiles: OutputtedDocsFiles) => { - const list = outputtedDocsFiles.map(formatLLMsDocLine).join('\n') - return `# Clerk\n\n## Docs\n\n${list}` + return `# Clerk\n\n${sections.filter((section) => section.length > 0).join('\n\n')}` } export const normalizeFrontmatterDescription = (raw: unknown): string | undefined => {