From 9eb0a88e09c94ae6d18c1a8667ab6851571beb91 Mon Sep 17 00:00:00 2001 From: Nick Wylynko Date: Thu, 7 May 2026 13:13:11 -0400 Subject: [PATCH 1/3] add .md to the end of the guide links in llms.txt --- scripts/build-docs.test.ts | 2 +- scripts/lib/llms.test.ts | 42 ++++++++++++++++++++++++++++++++++++++ scripts/lib/llms.ts | 14 ++++++++++--- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 scripts/lib/llms.test.ts diff --git a/scripts/build-docs.test.ts b/scripts/build-docs.test.ts index 1a68a40c1e..e2bc7c55e0 100644 --- a/scripts/build-docs.test.ts +++ b/scripts/build-docs.test.ts @@ -7422,7 +7422,7 @@ description: Generated API docs ## Docs -- [API Documentation]({{SITE_URL}}/docs/api-doc)`) +- [API Documentation]({{SITE_URL}}/docs/api-doc.md)`) }) test('Should output llms-full.txt full pages', async () => { diff --git a/scripts/lib/llms.test.ts b/scripts/lib/llms.test.ts new file mode 100644 index 0000000000..b67d9aa190 --- /dev/null +++ b/scripts/lib/llms.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from 'vitest' +import { buildLLMsDocsUrl } from './llms' + +const baseDocsLink = '/docs/' + +describe('buildLLMsDocsUrl', () => { + test('appends .md to top-level guide paths', () => { + expect(buildLLMsDocsUrl('cli.mdx', baseDocsLink)).toBe('{{SITE_URL}}/docs/cli.md') + }) + + test('appends .md to nested guide paths', () => { + expect(buildLLMsDocsUrl('nextjs/getting-started/quickstart.mdx', baseDocsLink)).toBe( + '{{SITE_URL}}/docs/nextjs/getting-started/quickstart.md', + ) + }) + + test('keeps root index as /docs/index.md', () => { + expect(buildLLMsDocsUrl('index.mdx', baseDocsLink)).toBe('{{SITE_URL}}/docs/index.md') + }) + + test('collapses nested /index segments to the parent slug', () => { + expect(buildLLMsDocsUrl('guides/index.mdx', baseDocsLink)).toBe('{{SITE_URL}}/docs/guides.md') + }) + + test('handles deeply nested /index paths', () => { + expect(buildLLMsDocsUrl('nextjs/guides/billing/index.mdx', baseDocsLink)).toBe( + '{{SITE_URL}}/docs/nextjs/guides/billing.md', + ) + }) + + test('respects a custom baseDocsLink', () => { + expect(buildLLMsDocsUrl('cli.mdx', '/docs/pr/feature-branch/')).toBe( + '{{SITE_URL}}/docs/pr/feature-branch/cli.md', + ) + }) + + test('does not strip a non-trailing index segment', () => { + expect(buildLLMsDocsUrl('index/getting-started.mdx', baseDocsLink)).toBe( + '{{SITE_URL}}/docs/index/getting-started.md', + ) + }) +}) diff --git a/scripts/lib/llms.ts b/scripts/lib/llms.ts index 50d349f1e4..d4884e806f 100644 --- a/scripts/lib/llms.ts +++ b/scripts/lib/llms.ts @@ -13,6 +13,16 @@ export const writeLLMs = async (outputtedDocsFiles: OutputtedDocsFiles) => { return `# Clerk\n\n## Docs\n\n${list}` } +// Build the public markdown URL for a docs file path. Points at the `.md` +// export route so LLMs consuming llms.txt fetch the markdown variant directly. +export const buildLLMsDocsUrl = (filePath: string, baseDocsLink: string): string => { + // For nested index pages (e.g. `guides/index.mdx`) the canonical URL drops + // the trailing `/index`, matching the docs site's routing. The root + // `index.mdx` keeps its slug so it maps to `/docs/index.md`. + const slug = removeMdxSuffix(filePath).replace(/\/index$/, '') + return `{{SITE_URL}}${baseDocsLink}${slug}.md` +} + export const listOutputDocsFiles = (config: BuildConfig, docs: Docs, files: { path: string }[]) => { return files .filter(({ path }) => !path.startsWith('~/')) // Exclude these quick redirect pages @@ -25,9 +35,7 @@ export const listOutputDocsFiles = (config: BuildConfig, docs: Docs, files: { pa return { path, - url: `{{SITE_URL}}${config.baseDocsLink}${removeMdxSuffix(path) - .replace(/^index$/, '') // remove root index - .replace(/\/index$/, '')}`, // remove /index from the end, + url: buildLLMsDocsUrl(path, config.baseDocsLink), content, } }) From ea434f9b6c48f589b0949579464f527bbcc9a9c6 Mon Sep 17 00:00:00 2001 From: Nick Wylynko Date: Thu, 7 May 2026 13:17:20 -0400 Subject: [PATCH 2/3] linting --- scripts/lib/llms.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/lib/llms.test.ts b/scripts/lib/llms.test.ts index b67d9aa190..87cce9c165 100644 --- a/scripts/lib/llms.test.ts +++ b/scripts/lib/llms.test.ts @@ -29,9 +29,7 @@ describe('buildLLMsDocsUrl', () => { }) test('respects a custom baseDocsLink', () => { - expect(buildLLMsDocsUrl('cli.mdx', '/docs/pr/feature-branch/')).toBe( - '{{SITE_URL}}/docs/pr/feature-branch/cli.md', - ) + expect(buildLLMsDocsUrl('cli.mdx', '/docs/pr/feature-branch/')).toBe('{{SITE_URL}}/docs/pr/feature-branch/cli.md') }) test('does not strip a non-trailing index segment', () => { From e970e4bc4b573a58e4f422bafc8d713587a38b14 Mon Sep 17 00:00:00 2001 From: Michael Novotny Date: Fri, 15 May 2026 20:00:57 -0500 Subject: [PATCH 3/3] Add build-level test for root index in llms.txt and drop unused destructured vars Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/build-docs.test.ts | 68 ++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/scripts/build-docs.test.ts b/scripts/build-docs.test.ts index 190067d624..f1774bcc49 100644 --- a/scripts/build-docs.test.ts +++ b/scripts/build-docs.test.ts @@ -243,7 +243,7 @@ Testing with a simple page.`) test('Warning on missing description in frontmatter', async () => { // Create temp environment with minimal files array - const { tempDir, pathJoin } = await createTempFiles([ + const { tempDir } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -1174,7 +1174,7 @@ Testing with a simple page.`, }) test('Invalid SDK in frontmatter fails the build', async () => { - const { tempDir, pathJoin } = await createTempFiles([ + const { tempDir } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -1745,7 +1745,7 @@ sdk: nextjs, react }) test('should handle components with both `sdk` and `notSdk` props', async () => { - const { tempDir, pathJoin } = await createTempFiles([ + const { tempDir } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -2455,7 +2455,7 @@ iOS Quickstart`, describe('Heading Validation', () => { test('should error on duplicate headings', async () => { - const { tempDir, pathJoin } = await createTempFiles([ + const { tempDir } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -2785,7 +2785,7 @@ title: Simple Test }) test('Nested partials should work (partial inside a partial)', async () => { - const { tempDir, pathJoin, readFile } = await createTempFiles([ + const { tempDir, readFile } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -4276,7 +4276,7 @@ title: Core Page }) test('should correctly handle links with anchors to specific sections of documents', async () => { - const { tempDir, pathJoin } = await createTempFiles([ + const { tempDir } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -5087,7 +5087,7 @@ sourceFile: /docs/doc-2.mdx }) test('Should embed links in sdk scoped docs', async () => { - const { tempDir, readFile, listFiles } = await createTempFiles([ + const { tempDir, readFile } = await createTempFiles([ { path: './docs/manifest.json', content: JSON.stringify({ @@ -7511,6 +7511,60 @@ description: Generated API docs - [API Documentation]({{SITE_URL}}/docs/api-doc.md): Generated API docs`) }) + test('Should collapse /index in llms.txt URLs', async () => { + const { tempDir, readFile } = await createTempFiles([ + { + path: './docs/manifest.json', + content: JSON.stringify({ + navigation: [ + [ + { title: 'Home', href: '/docs/index' }, + { title: 'Guides', href: '/docs/guides/index' }, + ], + ], + }), + }, + { + path: './docs/index.mdx', + content: `--- +title: Home +description: Welcome to the docs +--- + +# Welcome +`, + }, + { + path: './docs/guides/index.mdx', + content: `--- +title: Guides +description: Guides overview +--- + +# Guides +`, + }, + ]) + + await build( + await createConfig({ + ...baseConfig, + basePath: tempDir, + validSdks: ['react'], + llms: { + overviewPath: 'llms.txt', + }, + }), + ) + + expect(await readFile('./dist/llms.txt')).toEqual(`# Clerk + +## Docs + +- [Home]({{SITE_URL}}/docs.md): Welcome to the docs +- [Guides]({{SITE_URL}}/docs/guides.md): Guides overview`) + }) + test('Should output llms-full.txt full pages', async () => { const { tempDir, readFile } = await createTempFiles([ {