feat: improve agent-friendly docs score (AFDocs quick wins)

rachaelrenk · oz-agent · rachaelrenk · commit 747e9357c7f1 · 2026-05-05T09:19:26.000-10:00
- Add llms.txt directive to all HTML pages (visually-hidden div in header)
  and all generated .md files (blockquote at top)
- Add Astro middleware for Accept: text/markdown content negotiation,
  using existing shouldServeMarkdown() helper
- Fix llms.txt coverage: university/** → guides/** (directory was renamed),
  add community pages to Support customSet
- Add MCP discovery file at .well-known/mcp.json pointing to Kapa MCP
- Verify &lt;link rel=alternate type=text/markdown&gt; already appears early
  in &lt;head&gt; — no repositioning needed

Checks addressed:
  llms-txt-directive-html (FAIL → PASS)
  llms-txt-directive-md (FAIL → PASS)
  content-negotiation (FAIL → PASS)
  content-start-position (mitigated via Tasks 1+2)
  llms-txt-coverage (80% → improved)
  MCP Server Discoverable (FAIL → PASS)

Co-Authored-By: Oz &lt;oz-agent@warp.dev&gt;
diff --git a/astro.config.mjs b/astro.config.mjs
@@ -139,9 +139,9 @@ export default defineConfig({
 				// widely consumed by AI agents.
 				starlightLlmsTxt({
 					projectName: 'Warp',
-					// Excludes open-source-licenses from llms-full.txt and llms-small.txt.
-					// The file is ~25k lines and causes a stack overflow in hast-util-to-text.
-					// llms-custom sets exclude it separately via explicit path enumeration below.
+					// Excludes pages from llms-small.txt that cause a stack overflow
+					// in hast-util-to-text due to their size. Note: the `exclude`
+					// option only applies to llms-small.txt, not llms-full.txt.
 					exclude: ['support-and-community/community/open-source-licenses'],
 					description:
 						'Documentation for Warp, the agentic development environment, and Oz, Warp\'s programmable agent for running and coordinating agents at scale.',
@@ -153,10 +153,13 @@ export default defineConfig({
 						{ label: 'Getting Started', description: 'Installation, quickstart, and migration guides.', paths: ['getting-started/**'] },
 						{ label: 'Knowledge and Collaboration', description: 'Warp Drive, teams, and the Admin Panel.', paths: ['knowledge-and-collaboration/**'] },
 						{ label: 'Reference', description: 'CLI and API reference.', paths: ['reference/**'] },
-					// Excludes support-and-community/community/ — open-source-licenses.mdx is ~25k
-					// lines and causes a stack overflow in hast-util-to-text during llms-txt generation.
-					{ label: 'Support', description: 'Troubleshooting, billing, and privacy.', paths: ['support-and-community/index', 'support-and-community/plans-and-billing/**', 'support-and-community/privacy-and-security/**', 'support-and-community/troubleshooting-and-support/**'] },
-						{ label: 'Guides (Warp University)', description: 'Task-oriented walkthroughs.', paths: ['university/**'] },
+				// Includes all support-and-community/ pages except open-source-licenses.mdx
+				// (excluded globally above — ~25k lines causes a stack overflow in hast-util-to-text).
+				{ label: 'Support', description: 'Troubleshooting, billing, and privacy.', paths: ['support-and-community/index', 'support-and-community/plans-and-billing/**', 'support-and-community/privacy-and-security/**', 'support-and-community/troubleshooting-and-support/**', 'support-and-community/community/contributing', 'support-and-community/community/open-source-partnership', 'support-and-community/community/refer-a-friend'] },
+					{ label: 'Guides', description: 'Task-oriented walkthroughs and tutorials.', paths: ['guides/**'] },
+					// Changelog excluded — the single page (4k lines) causes a stack overflow
+					// in hast-util-to-text during llms-full.txt generation. The page is still
+					// available at /changelog/ and indexed by the sitemap.
 					],
 				}),
 			],
diff --git a/public/.well-known/mcp.json b/public/.well-known/mcp.json
@@ -0,0 +1,6 @@
+{
+  "name": "Warp Documentation",
+  "description": "Search and retrieve Warp documentation — the agentic development environment and Oz, Warp's programmable agent platform.",
+  "url": "https://warp.mcp.kapa.ai/sse",
+  "transport": "sse"
+}
diff --git a/src/components/CustomHeader.astro b/src/components/CustomHeader.astro
@@ -18,6 +18,10 @@ import ThemeSelect from 'virtual:starlight/components/ThemeSelect';
 import WarpTopicNav from './WarpTopicNav.astro';
 ---
 
+<div class="llms-directive sr-only" aria-hidden="true">
+	For the complete documentation in markdown, see <a href="/llms.txt">llms.txt</a>.
+	Markdown versions of each page are available by appending .md to any URL.
+</div>
 <div class="header">
 	<div class="title-wrapper sl-flex">
 		<SiteTitle />
diff --git a/src/integrations/docs-markdown-integration.js b/src/integrations/docs-markdown-integration.js
@@ -88,7 +88,10 @@ function convertHtmlToMarkdown(html) {
 	const clone = /** @type {HTMLElement} */ (contentRoot.cloneNode(true));
 	sanitizeRoot(clone);
 	const markdownBody = turndown.turndown(clone.innerHTML).trim();
-	const sections = [`# ${normalizeWhitespace(title)}`];
+	const llmsDirective =
+		'> For the complete documentation index, see [llms.txt](/llms.txt).\n' +
+		'> Markdown versions of each page are available by appending .md to any URL.';
+	const sections = [llmsDirective, `# ${normalizeWhitespace(title)}`];
 
 	if (description) {
 		sections.push(description);
diff --git a/src/middleware.ts b/src/middleware.ts
@@ -0,0 +1,32 @@
+import { defineMiddleware } from 'astro:middleware';
+import {
+	shouldServeMarkdown,
+	isEligibleDocHtmlPath,
+	getMarkdownPathFromHtmlPath,
+} from './lib/docs-markdown.js';
+
+/**
+ * Content negotiation middleware.
+ *
+ * When an agent sends `Accept: text/markdown` (or is identified by user-agent),
+ * rewrite the request to the pre-rendered `.md` variant of the page. This lets
+ * agents like Claude Code, Cursor, and OpenCode get clean markdown without
+ * needing to discover the `.md` URL convention first.
+ *
+ * The `shouldServeMarkdown` helper handles both explicit Accept-header
+ * negotiation and user-agent fallback detection (see `src/lib/docs-markdown.js`).
+ */
+export const onRequest = defineMiddleware(async (context, next) => {
+	const { request, url } = context;
+
+	if (!isEligibleDocHtmlPath(url.pathname)) {
+		return next();
+	}
+
+	if (!shouldServeMarkdown(request)) {
+		return next();
+	}
+
+	const mdPath = getMarkdownPathFromHtmlPath(url.pathname);
+	return context.rewrite(mdPath);
+});
