Group SDK-scoped pages under sub-headers in /docs/llms.txt

scripts/build-docs.test.ts

@@ -7511,6 +7511,170 @@ description: Generated API docs
 - [API Documentation]({{SITE_URL}}/docs/api-doc): Generated API docs`)
   })
 
+  test('Should group SDK-scoped pages under sub-headers in llms.txt overview', async () => {
+    const { tempDir, readFile } = await createTempFiles([
+      {
+        path: './docs/manifest.json',
+        content: JSON.stringify({
+          navigation: [
+            [
+              { title: 'Generic Doc', href: '/docs/generic-doc' },
+              { title: 'SDK Doc', href: '/docs/sdk-doc' },
+            ],
+          ],
+        }),
+      },
+      {
+        path: './docs/generic-doc.mdx',
+        content: `---
+title: Generic Doc
+description: A generic guide
+---
+
+# Generic Doc
+`,
+      },
+      {
+        path: './docs/sdk-doc.mdx',
+        content: `---
+title: SDK Doc
+description: An SDK-scoped guide
+sdk: nextjs, react
+---
+
+# SDK Doc
+`,
+      },
+    ])
+
+    await build(
+      await createConfig({
+        ...baseConfig,
+        basePath: tempDir,
+        validSdks: ['nextjs', 'react'],
+        llms: {
+          overviewPath: 'llms.txt',
+        },
+      }),
+    )
+
+    expect(await readFile('./dist/llms.txt')).toEqual(`# Clerk
+
+## Docs
+
+- [Generic Doc]({{SITE_URL}}/docs/generic-doc): A generic guide
+
+### Next.js
+
+- [SDK Doc]({{SITE_URL}}/docs/nextjs/sdk-doc): An SDK-scoped guide
+
+### React
+
+- [SDK Doc]({{SITE_URL}}/docs/react/sdk-doc): An SDK-scoped guide`)
+  })
+
+  test('Should group reference/<sdk>/ pages and URL-aliased SDKs under their SDK sub-header', async () => {
+    const { tempDir, readFile } = await createTempFiles([
+      {
+        path: './docs/manifest.json',
+        content: JSON.stringify({
+          navigation: [
+            [
+              { title: 'Generic Guide', href: '/docs/generic-guide' },
+              { title: 'Vue Plugin', href: '/docs/reference/vue/clerk-plugin' },
+              { title: 'Astro Middleware', href: '/docs/reference/astro/clerk-middleware' },
+              { title: 'JS Overview', href: '/docs/reference/javascript/overview' },
+              { title: 'Express Middleware', href: '/docs/reference/express/clerk-middleware' },
+            ],
+          ],
+        }),
+      },
+      {
+        path: './docs/generic-guide.mdx',
+        content: `---
+title: Generic Guide
+description: A generic guide
+---
+
+# Generic Guide
+`,
+      },
+      {
+        path: './docs/reference/vue/clerk-plugin.mdx',
+        content: `---
+title: clerkPlugin
+description: Vue clerkPlugin reference
+---
+
+# clerkPlugin
+`,
+      },
+      {
+        path: './docs/reference/astro/clerk-middleware.mdx',
+        content: `---
+title: clerkMiddleware (Astro)
+description: Astro middleware reference
+---
+
+# clerkMiddleware
+`,
+      },
+      {
+        path: './docs/reference/javascript/overview.mdx',
+        content: `---
+title: JavaScript Overview
+description: JS frontend SDK overview
+---
+
+# JavaScript Overview
+`,
+      },
+      {
+        path: './docs/reference/express/clerk-middleware.mdx',
+        content: `---
+title: clerkMiddleware (Express)
+description: Express middleware reference
+---
+
+# clerkMiddleware
+`,
+      },
+    ])
+
+    await build(
+      await createConfig({
+        ...baseConfig,
+        basePath: tempDir,
+        validSdks: ['vue', 'astro', 'js-frontend', 'expressjs'],
+        llms: {
+          overviewPath: 'llms.txt',
+        },
+      }),
+    )
+
+    expect(await readFile('./dist/llms.txt')).toEqual(`# Clerk
+
+## Docs
+
+- [Generic Guide]({{SITE_URL}}/docs/generic-guide): A generic guide
+
+### Vue
+
+- [clerkPlugin]({{SITE_URL}}/docs/reference/vue/clerk-plugin): Vue clerkPlugin reference
+
+### Astro
+
+- [clerkMiddleware (Astro)]({{SITE_URL}}/docs/reference/astro/clerk-middleware): Astro middleware reference
+
+### JavaScript
+
+- [JavaScript Overview]({{SITE_URL}}/docs/reference/javascript/overview): JS frontend SDK overview
+
+### Express
+
+- [clerkMiddleware (Express)]({{SITE_URL}}/docs/reference/express/clerk-middleware): Express middleware reference`)
+  })
+
   test('Should output llms-full.txt full pages', async () => {
     const { tempDir, readFile } = await createTempFiles([
       {

scripts/build-docs.ts

@@ -1212,7 +1212,7 @@ ${yaml.stringify({
     }
 
     if (config.llms?.overviewPath) {
-      const llms = await generateLLMs(outputtedDocsFiles)
+      const llms = await generateLLMs(outputtedDocsFiles, config.validSdks)
       await writeFile(config.llms.overviewPath, llms)
     }
   }

scripts/lib/llms.test.ts

@@ -8,6 +8,8 @@ describe('formatLLMsDocLine', () => {
         title: 'Quickstart',
         url: 'https://example.com/docs/quickstart',
         description: 'Get up and running with Clerk in minutes.',
+        path: 'quickstart.mdx',
+        content: '',
       }),
     ).toBe('- [Quickstart](https://example.com/docs/quickstart): Get up and running with Clerk in minutes.')
   })
@@ -17,6 +19,9 @@ describe('formatLLMsDocLine', () => {
       formatLLMsDocLine({
         title: 'Quickstart',
         url: 'https://example.com/docs/quickstart',
+        description: undefined,
+        path: 'quickstart.mdx',
+        content: '',
       }),
     ).toBe('- [Quickstart](https://example.com/docs/quickstart)')
   })
@@ -168,22 +173,25 @@ Body`,
 
 describe('writeLLMs', () => {
   test('renders a markdown list with descriptions when available', async () => {
-    const result = await writeLLMs([
-      {
-        path: 'quickstart.mdx',
-        url: '{{SITE_URL}}/docs/quickstart',
-        content: '',
-        title: 'Quickstart',
-        description: 'Get up and running with Clerk in minutes.',
-      },
-      {
-        path: 'overview.mdx',
-        url: '{{SITE_URL}}/docs/overview',
-        content: '',
-        title: 'Overview',
-        description: undefined,
-      },
-    ])
+    const result = await writeLLMs(
+      [
+        {
+          path: 'quickstart.mdx',
+          url: '{{SITE_URL}}/docs/quickstart',
+          content: '',
+          title: 'Quickstart',
+          description: 'Get up and running with Clerk in minutes.',
+        },
+        {
+          path: 'overview.mdx',
+          url: '{{SITE_URL}}/docs/overview',
+          content: '',
+          title: 'Overview',
+          description: undefined,
+        },
+      ],
+      ['react'],
+    )
 
     expect(result).toBe(
       [

scripts/lib/llms.ts

@@ -1,18 +1,89 @@
 import yaml from 'yaml'
+import type { SDK } from './schemas'
 
 type Docs = Map<string, string>
 
+// Display names for SDKs when rendered as sub-headers in llms.txt.
+// Keep these in sync with VALID_SDKS in ./schemas.ts.
+const SDK_DISPLAY_NAMES: Record<SDK, string> = {
+  nextjs: 'Next.js',
+  react: 'React',
+  'js-frontend': 'JavaScript',
+  'chrome-extension': 'Chrome Extension',
+  expo: 'Expo',
+  android: 'Android',
+  ios: 'iOS',
+  expressjs: 'Express',
+  fastify: 'Fastify',
+  'react-router': 'React Router',
+  'tanstack-react-start': 'TanStack React Start',
+  go: 'Go',
+  astro: 'Astro',
+  nuxt: 'Nuxt',
+  vue: 'Vue',
+  ruby: 'Ruby',
+}
+
+// Some SDKs are referenced in URL/file paths under a slug that doesn't match
+// their SDK key (e.g. /docs/reference/javascript/... is the js-frontend SDK).
+// This map allows those path segments to be recognized as SDK-scoped.
+const PATH_SEGMENT_SDK_ALIASES: Record<string, SDK> = {
+  javascript: 'js-frontend',
+  express: 'expressjs',
+}
+
+const getSdkFromPath = (path: string, validSdks: readonly SDK[]): SDK | null => {
+  // Skip the trailing file segment (e.g. "expo.mdx") - we only want directory
+  // segments, so a generic doc whose filename contains an SDK name is not
+  // misclassified as SDK-scoped.
+  const segments = path.split('/').slice(0, -1)
+  for (const segment of segments) {
+    if (validSdks.includes(segment as SDK)) {
+      return segment as SDK
+    }
+    const aliased = PATH_SEGMENT_SDK_ALIASES[segment]
+    if (aliased && validSdks.includes(aliased)) {
+      return aliased
+    }
+  }
+  return null
+}
+
+const getSdkDisplayName = (sdk: SDK): string => SDK_DISPLAY_NAMES[sdk] ?? sdk
+
 export const writeLLMsFull = async (outputtedDocsFiles: OutputtedDocsFiles) => {
   return outputtedDocsFiles.map((file) => file.content).join('\n')
 }
 
-export const formatLLMsDocLine = (page: { title: string; url: string; description?: string }) => {
-  return page.description ? `- [${page.title}](${page.url}): ${page.description}` : `- [${page.title}](${page.url})`
-}
+export const formatLLMsDocLine = (page: OutputtedDocsFiles[number]) =>
+  page.description ? `- [${page.title}](${page.url}): ${page.description}` : `- [${page.title}](${page.url})`
+
+export const writeLLMs = async (outputtedDocsFiles: OutputtedDocsFiles, validSdks: readonly SDK[]) => {
+  const generic: OutputtedDocsFiles = []
+  const bySdk = new Map<SDK, OutputtedDocsFiles>()
+
+  for (const page of outputtedDocsFiles) {
+    const sdk = getSdkFromPath(page.path, validSdks)
+    if (sdk === null) {
+      generic.push(page)
+    } else {
+      const list = bySdk.get(sdk) ?? []
+      list.push(page)
+      bySdk.set(sdk, list)
+    }
+  }
+
+  const sections: string[] = [`## Docs`, generic.map(formatLLMsDocLine).join('\n')]
+
+  // Emit SDK sections in the order they appear in validSdks for stable, predictable output.
+  for (const sdk of validSdks) {
+    const pages = bySdk.get(sdk)
+    if (!pages || pages.length === 0) continue
+    sections.push(`### ${getSdkDisplayName(sdk)}`)
+    sections.push(pages.map(formatLLMsDocLine).join('\n'))
+  }
 
-export const writeLLMs = async (outputtedDocsFiles: OutputtedDocsFiles) => {
-  const list = outputtedDocsFiles.map(formatLLMsDocLine).join('\n')
-  return `# Clerk\n\n## Docs\n\n${list}`
+  return `# Clerk\n\n${sections.filter((section) => section.length > 0).join('\n\n')}`
 }
 
 export const normalizeFrontmatterDescription = (raw: unknown): string | undefined => {