Skip to content

Serve markdown indexes for the docs root and section hubs to agents #4996

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
philip wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Serve markdown indexes for the docs root and section hubs to agents #4996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions next.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,18 @@ const defaultConfig = {
],
},
async headers() {
// Serve top-level index URLs (e.g. /faqs.md, /programs.md, /docs.md) as inline markdown.
// Only non-nested routes have an index file; /docs.md is aliased to /docs/llms.txt via rewrite.
const mdIndexHeaders = Object.keys(CONTENT_ROUTES)
.filter((route) => !route.includes('/'))
.map((route) => ({
source: `/${route}.md`,
headers: [
{ key: 'Content-Disposition', value: 'inline' },
{ key: 'Content-Type', value: 'text/markdown; charset=utf-8' },
],
}));

return [
{
source: '/',
Expand Down Expand Up @@ -136,7 +148,9 @@ const defaultConfig = {
],
},
{
source: '/(docs|postgresql|guides|branching|programs|use-cases|faqs)/:path*.md',
source: `/(${Object.keys(CONTENT_ROUTES)
.filter((r) => !r.includes('/'))
.join('|')})/:path*.md`,
headers: [
{
key: 'Content-Disposition',
Expand All @@ -148,6 +162,7 @@ const defaultConfig = {
},
],
},
...mdIndexHeaders,
];
},
async redirects() {
Expand Down Expand Up @@ -2053,11 +2068,10 @@ const defaultConfig = {
destination: '/docs/connect/connection-errors',
permanent: true,
},
{
source: '/docs',
destination: '/docs/introduction',
permanent: true,
},
// NOTE: bare `/docs` is intentionally NOT redirected here. The middleware
// (src/proxy.js) owns it: agents / Accept: markdown get /docs/llms.txt, and
// browsers are redirected to /docs/introduction. A next.config redirect would
// run before middleware and intercept the markdown case.
{
source: '/docs/postgres',
destination: '/docs/postgres/index',
Expand Down Expand Up @@ -2472,8 +2486,10 @@ const defaultConfig = {

// /:path*.md above requires at least one segment after the route name,
// so /branching.md (no separator) doesn't match. Add explicit index rewrites.
// /docs.md is excluded here: it's aliased to the canonical /docs/llms.txt index
// below rather than a generated page-listing (see process-md-for-llms.js).
const indexRewrites = Object.keys(CONTENT_ROUTES)
.filter((route) => !route.includes('/'))
.filter((route) => !route.includes('/') && route !== 'docs')
.map((route) => ({
source: `/${route}.md`,
destination: `/md/${route}.md`,
Expand All @@ -2498,6 +2514,12 @@ const defaultConfig = {
source: '/docs/.well-known/skills/:name/SKILL.md',
destination: '/docs/ai/skills/:name/SKILL.md',
},
// /docs.md serves the canonical, curated docs index (llms.txt) instead of a
// generated page-listing. beforeFiles so the [slug] catch-all doesn't intercept it.
{ source: '/docs.md', destination: '/docs/llms.txt' },
// Index .md files (e.g. /faqs.md, /programs.md) must be beforeFiles so the
// top-level [slug] catch-all doesn't intercept them before the rewrite fires.
...indexRewrites,
],
// afterFiles: runs after checking pages/public files but before dynamic routes
// This ensures physical .md files are served first, with fallback to public/md/
Expand All @@ -2518,7 +2540,6 @@ const defaultConfig = {
},
{ source: '/skill.md', destination: '/docs/ai/skills/neon-postgres/SKILL.md' },
{ source: '/docs/changelog/:path*.md', destination: '/md/changelog/:path*.md' },
...indexRewrites,
...contentRewrites,
],
// fallback: existing rewrites for external services
Expand Down
165 changes: 138 additions & 27 deletions scripts/test-markdown-urls.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,20 +375,20 @@ function buildTests() {
);

if (hasDotMd) {
// No index file at public/md/guides.md etc.; expect same contract as other .md 404s
// (requires top-level .md routing — see PR #4735).
// /{route}.md serves the generated index file from public/md/{route}.md —
// a listing of all pages in that route. Returns 200 markdown even though
// /{route} (without .md) is excluded from middleware markdown serving.
add(
'Excluded route',
path,
'dot-md',
[
(r) => expectStatus(r.status, 404),
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectBodyContains(r.body, 'Page Not Found'),
(r) => expectMarkdownBody(r.body),
(r) => expectBodyContains(r.body, '/docs/llms.txt'),
(r) => expectHeader(r.headers, 'x-content-source', 'md-404'),
],
{ note: 'hub index .md → markdown 404 (not Vercel HTML 404)' }
{ note: 'hub index .md → generated index listing (200 markdown)' }
);
}
}
Expand Down Expand Up @@ -639,66 +639,177 @@ function buildTests() {
);
}

// ── 9b. Blog post (not a docs content route; stays HTML for agents / Accept: markdown)
// Example: https://neon.com/blog/prewarming
// ── 9b. Non-docs marketing page (stays HTML for agents / Accept: markdown)
// Tests that middleware does not serve markdown for pages outside CONTENT_ROUTES.

const blogPostPath = '/blog/prewarming';
const nonDocsPath = '/about-us';

add(
'Blog post',
blogPostPath,
'Non-docs page',
nonDocsPath,
'browser',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/html'),
(r) => expectHtmlBody(r.body),
],
{
spotCheck: (r) => expectBodyContains(r.body, 'Prewarming', true),
note: 'engineering blog article',
}
{ note: 'marketing page, not a content route' }
);

add(
'Blog post',
blogPostPath,
'Non-docs page',
nonDocsPath,
'accept-md',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/html'),
(r) => expectHtmlBody(r.body),
(r) => {
const src = r.headers.get('x-content-source');
if (src === 'markdown') return 'blog post returned x-content-source: markdown';
if (src === 'markdown') return 'non-docs page returned x-content-source: markdown';
return null;
},
],
{
spotCheck: (r) => expectBodyContains(r.body, 'Prewarming', true),
note: 'Accept: markdown must not serve docs markdown',
}
{ note: 'Accept: markdown must not serve docs markdown' }
);

add(
'Blog post',
blogPostPath,
'Non-docs page',
nonDocsPath,
'agent-ua',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/html'),
(r) => expectHtmlBody(r.body),
(r) => {
const src = r.headers.get('x-content-source');
if (src === 'markdown') return 'blog post returned x-content-source: markdown';
if (src === 'markdown') return 'non-docs page returned x-content-source: markdown';
return null;
},
],
{ note: 'AI UA must not get docs markdown' }
);

// ── 9c. Hub index .md files — generated by generateRouteIndex at postbuild ──
// These serve a listing of all pages in that route (requires --generate or postbuild).

const hubIndexRoutes = [
{ path: '/faqs.md', spotWord: 'FAQs' },
{ path: '/programs.md', spotWord: 'Programs' },
{ path: '/branching.md', spotWord: 'Branching' },
{ path: '/guides.md', spotWord: 'Guides' },
{ path: '/postgresql.md', spotWord: 'PostgreSQL' },
];

for (const { path, spotWord } of hubIndexRoutes) {
// dot-md (browser UA): served as static file via beforeFiles rewrite
add(
'Hub index .md',
path,
'browser',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectMarkdownBody(r.body),
(r) => expectBodyContains(r.body, '/docs/llms.txt'),
],
{ spotCheck: (r) => expectBodyContains(r.body, spotWord, true), note: 'static index listing' }
);

// agent-ua: middleware fetches /md/{route}.md and serves it
add(
'Hub index .md',
path,
'agent-ua',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectMarkdownBody(r.body),
],
{
spotCheck: (r) => expectBodyContains(r.body, spotWord, true),
note: 'agent gets index listing',
}
);
}

// ── 9d. /docs.md — aliased to the canonical curated index (llms.txt), not a
// generated page-listing. Both the static rewrite (beforeFiles) and middleware
// (CUSTOM_MARKDOWN_PATHS) resolve it to /docs/llms.txt.

add(
'Docs index .md',
'/docs.md',
'browser',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectMarkdownBody(r.body),
],
{
spotCheck: (r) => expectBodyContains(r.body, 'Prewarming', true),
note: 'AI UA must not get docs markdown',
spotCheck: (r) => expectBodyContains(r.body, '# Neon Postgres', true),
note: 'aliased to /docs/llms.txt',
}
);

add(
'Docs index .md',
'/docs.md',
'agent-ua',
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectMarkdownBody(r.body),
],
{
spotCheck: (r) => expectBodyContains(r.body, '# Neon Postgres', true),
note: 'agent gets llms.txt content',
}
);

// ── 9e. Bare /docs — content-negotiated by the middleware (src/proxy.js). The
// /docs→/docs/introduction redirect lives in middleware (not next.config) so it runs
// AFTER the markdown check: agents / non-HTML Accept get llms.txt; browsers redirect.

// Browser (Accept: text/html, normal UA): redirected to the introduction page.
add(
'Bare /docs',
'/docs',
'browser',
[
(r) => expectStatus(r.status, 308),
(r) => expectHeader(r.headers, 'location', '/docs/introduction'),
(r) => expectHeader(r.headers, 'vary', 'Accept'),
],
{ note: 'browser → redirect to /docs/introduction (Vary: Accept)' }
);

// Markdown-negotiated requests all resolve to /docs/llms.txt with the doc headers.
// Each mode exercises a different branch of isAIAgentRequest():
// accept-md → Accept: text/markdown
// accept-plain→ prefersNonHtml (no text/html in Accept)
// agent-ua → known agent User-Agent (Claude)
// agent-axios → HTTP-client User-Agent (axios) with Accept: text/html
for (const mode of ['accept-md', 'accept-plain', 'agent-ua', 'agent-axios']) {
add(
'Bare /docs',
'/docs',
mode,
[
(r) => expectStatus(r.status, 200),
(r) => expectContentType(r.contentType, 'text/markdown'),
(r) => expectMarkdownBody(r.body),
(r) => expectHeader(r.headers, 'x-content-source', 'markdown'),
(r) => expectHeader(r.headers, 'x-robots-tag', 'noindex'),
(r) => expectHeader(r.headers, 'vary', 'Accept'),
],
{
spotCheck: (r) => expectBodyContains(r.body, '# Neon Postgres', true),
note: 'markdown client → /docs/llms.txt content',
}
);
}

// ── 9. Individual changelog entry (file must exist under public/md/changelog/ — run --generate)
// ---------------------------------------------------------------------------

Expand Down
8 changes: 1 addition & 7 deletions src/app/(docs)/docs/[...slug]/page.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { notFound } from 'next/navigation';

import Post from 'components/pages/doc/post';
import VERCEL_URL from 'constants/base';
import { DOCS_DIR_PATH, CHANGELOG_DIR_PATH } from 'constants/content';
import { DOCS_DIR_PATH, CHANGELOG_DIR_PATH, isUnusedOrSharedContent } from 'constants/content';
import LINKS from 'constants/links';
import { getPostBySlug } from 'utils/api-content';
import { getAllPosts, getAllChangelogs, getNavigationLinks, getNavigation } from 'utils/api-docs';
Expand All @@ -12,12 +12,6 @@ import { getFlatSidebar } from 'utils/get-flat-sidebar';
import getMetadata from 'utils/get-metadata';
import getTableOfContents from 'utils/get-table-of-contents';

const isUnusedOrSharedContent = (slug) =>
slug.includes('unused/') ||
slug.includes('shared-content/') ||
slug.includes('README') ||
slug.includes('GUIDE_TEMPLATE');

export async function generateStaticParams() {
const posts = await getAllPosts();

Expand Down
8 changes: 1 addition & 7 deletions src/app/postgresql/[...slug]/page.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { notFound } from 'next/navigation';

import Post from 'components/pages/doc/post';
import VERCEL_URL from 'constants/base';
import { POSTGRESQL_DIR_PATH } from 'constants/content';
import { POSTGRESQL_DIR_PATH, isUnusedOrSharedContent } from 'constants/content';
import { POSTGRESQL_BASE_PATH } from 'constants/docs';
import { getPostBySlug } from 'utils/api-content';
import { getNavigation, getAllPostgresTutorials, getNavigationLinks } from 'utils/api-postgresql';
Expand All @@ -12,12 +12,6 @@ import { getFlatSidebar } from 'utils/get-flat-sidebar';
import getMetadata from 'utils/get-metadata';
import getTableOfContents from 'utils/get-table-of-contents';

const isUnusedOrSharedContent = (slug) =>
slug.includes('unused/') ||
slug.includes('shared-content/') ||
slug.includes('README') ||
slug.includes('GUIDE_TEMPLATE');

export async function generateStaticParams() {
const posts = await getAllPostgresTutorials();

Expand Down
10 changes: 10 additions & 0 deletions src/constants/content.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,21 @@ const EXCLUDED_ROUTES = [
'use-cases/serverless-apps',
];

const EXCLUDED_DIRS = ['shared-content', 'unused'];

const EXCLUDED_FILES = ['rss.xml', 'context7.json'];

const isUnusedOrSharedContent = (slug) =>
slug.includes('unused/') ||
slug.includes('shared-content/') ||
slug.includes('README') ||
slug.includes('GUIDE_TEMPLATE');

module.exports = {
CONTENT_ROUTES,
isUnusedOrSharedContent,
EXCLUDED_ROUTES,
EXCLUDED_DIRS,
EXCLUDED_FILES,
DOCS_DIR_PATH,
BRANCHING_DIR_PATH,
Expand Down
Loading