Fix AgentGrade content negotiation (#1946)

Th0rgal · web-flow · commit 84ea2d25d8af · 2026-06-04T09:31:31.000+01:00
* Fix AgentGrade content negotiation

* Prioritize markdown cues in docs proxy

* Refine root content negotiation
diff --git a/docs-site/agent-discovery.mjs b/docs-site/agent-discovery.mjs
@@ -6,6 +6,7 @@ export const AGENT_DISCOVERY_LINKS = [
 
 export const AGENT_DISCOVERY_HEADERS = [
   { key: "Link", value: AGENT_DISCOVERY_LINKS },
+  { key: "Vary", value: "Accept" },
   { key: "X-Llms-Txt", value: "/llms.txt" },
   { key: "X-Llms-Full-Txt", value: "/llms-full.txt" },
   { key: "X-Agent-Skill", value: "/skill.md" },
diff --git a/docs-site/app/api/agentgrade/root/route.ts b/docs-site/app/api/agentgrade/root/route.ts
@@ -0,0 +1,103 @@
+import { NextRequest, NextResponse } from "next/server";
+import { agentDiscoveryHeaderRecord } from "@/server/agent-discovery";
+import { negotiateContentType } from "@/server/content-negotiation";
+import { readDoc } from "@/server/docs";
+
+const description =
+  "Verity is a Lean-native language and formally verified compiler for Ethereum smart contracts.";
+
+const fallbackMarkdown = `# Verity
+
+> ${description}
+
+## Endpoints
+
+- [Documentation](https://veritylang.com/) - Verity overview and guides.
+- [Getting Started](https://veritylang.com/getting-started) - Local setup and first verification run.
+- [API Docs](https://veritylang.com/api/docs/_index) - Machine-readable documentation index.
+- [llms.txt](https://veritylang.com/llms.txt) - Agent operating manual.
+
+## Authentication
+
+No authentication is required for public documentation.
+`;
+
+const fallbackText = `${description}
+
+Documentation: https://veritylang.com/
+Getting started: https://veritylang.com/getting-started
+Machine-readable docs index: https://veritylang.com/api/docs/_index
+`;
+
+const html = `<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Verity</title>
+    <meta name="description" content="${description}">
+    <link rel="alternate" type="text/plain" title="llms.txt" href="/llms.txt">
+  </head>
+  <body>
+    <main>
+      <h1>Verity</h1>
+      <p>${description}</p>
+      <p><a href="/llms.txt">llms.txt</a></p>
+    </main>
+  </body>
+</html>
+`;
+
+export async function GET(request: NextRequest) {
+  const negotiatedType =
+    request.nextUrl.searchParams.get("type") ??
+    negotiateContentType(request.headers.get("accept"));
+  const doc = await readDoc("index");
+  const markdown = doc?.markdown ?? fallbackMarkdown;
+  const headers = {
+    ...agentDiscoveryHeaderRecord(),
+    "Vary": "Accept",
+    "X-Content-Type-Options": "nosniff",
+  };
+
+  if (negotiatedType === "application/json") {
+    return NextResponse.json(
+      {
+        name: "Verity",
+        url: "https://veritylang.com/",
+        description,
+        repository: "https://github.com/lfglabs-dev/verity",
+        docs: {
+          index: "https://veritylang.com/api/docs/_index",
+          llms: "https://veritylang.com/llms.txt",
+          full: "https://veritylang.com/llms-full.txt",
+        },
+      },
+      { headers }
+    );
+  }
+
+  if (negotiatedType === "text/plain") {
+    return new NextResponse(fallbackText, {
+      headers: {
+        ...headers,
+        "Content-Type": "text/plain; charset=utf-8",
+      },
+    });
+  }
+
+  if (negotiatedType === "text/html") {
+    return new NextResponse(html, {
+      headers: {
+        ...headers,
+        "Content-Type": "text/html; charset=utf-8",
+      },
+    });
+  }
+
+  return new NextResponse(markdown, {
+    headers: {
+      ...headers,
+      "Content-Type": "text/markdown; charset=utf-8",
+    },
+  });
+}
diff --git a/docs-site/app/layout.tsx b/docs-site/app/layout.tsx
@@ -127,7 +127,10 @@ const navbar = (
 export default async function RootLayout({ children }: { children: React.ReactNode }) {
   return (
     <html lang="en" dir="ltr" suppressHydrationWarning>
-      <Head />
+      <head>
+        <Head />
+        <link rel="alternate" type="text/plain" title="llms.txt" href="/llms.txt" />
+      </head>
       <body>
         <Layout
           navbar={navbar}
diff --git a/docs-site/proxy.ts b/docs-site/proxy.ts
@@ -1,5 +1,6 @@
 import { NextRequest, NextResponse } from "next/server";
 import { applyAgentDiscoveryHeaders } from "@/server/agent-discovery";
+import { negotiateContentType } from "@/server/content-negotiation";
 
 /**
  * Known LLM/AI user agent patterns
@@ -28,13 +29,18 @@ function isLLMUserAgent(userAgent: string | null): boolean {
   );
 }
 
+function withVaryAccept(response: NextResponse): NextResponse {
+  response.headers.set("Vary", "Accept");
+  return response;
+}
+
 /**
  * Proxy to serve raw markdown for AI agents
  *
  * Routes to raw markdown API when:
  * 1. URL ends with .md extension (e.g., /setup.md)
- * 2. Accept header includes text/markdown
- * 3. Accept header includes text/plain
+ * 2. Non-root Accept negotiation selects text/markdown
+ * 3. Non-root Accept negotiation selects text/plain
  * 4. Query param ?format=md is present
  * 5. User-Agent is a known LLM (ChatGPT, GPTBot, PerplexityBot, etc.)
  *
@@ -57,15 +63,19 @@ export function proxy(request: NextRequest) {
   }
 
   const userAgent = request.headers.get("User-Agent");
-
-  // Check if this is a docs page request that wants markdown
-  const wantsMarkdown =
+  const acceptHeader = request.headers.get("Accept");
+  const negotiatedType = negotiateContentType(acceptHeader);
+  const hasExplicitMarkdownCue =
     pathname.endsWith(".md") ||
-    request.headers.get("Accept")?.includes("text/markdown") ||
-    request.headers.get("Accept")?.includes("text/plain") ||
     request.nextUrl.searchParams.get("format") === "md" ||
     isLLMUserAgent(userAgent);
 
+  // Check if this is a docs page request that wants markdown
+  const wantsMarkdown =
+    hasExplicitMarkdownCue ||
+    (pathname !== "/" &&
+      (negotiatedType === "text/markdown" || negotiatedType === "text/plain"));
+
   if (wantsMarkdown) {
     // Normalize the path (remove .md extension if present)
     let docPath = pathname.replace(/\.md$/, "");
@@ -81,10 +91,31 @@ export function proxy(request: NextRequest) {
     url.searchParams.delete("format");
 
     // Rewrite to the API route (internal redirect, URL doesn't change for client)
-    return applyAgentDiscoveryHeaders(NextResponse.rewrite(url));
+    return withVaryAccept(applyAgentDiscoveryHeaders(NextResponse.rewrite(url)));
+  }
+
+  if (pathname === "/" && negotiatedType !== "text/html") {
+    const url = new URL(request.url);
+    url.pathname = "/api/agentgrade/root";
+    url.searchParams.set("type", negotiatedType);
+    return withVaryAccept(applyAgentDiscoveryHeaders(NextResponse.rewrite(url)));
+  }
+
+  if (pathname !== "/" && negotiatedType === "application/json") {
+    return withVaryAccept(
+      applyAgentDiscoveryHeaders(
+        NextResponse.json(
+          {
+            error: "not_found",
+            path: pathname,
+          },
+          { status: 404 }
+        )
+      )
+    );
   }
 
-  return applyAgentDiscoveryHeaders(NextResponse.next());
+  return withVaryAccept(applyAgentDiscoveryHeaders(NextResponse.next()));
 }
 
 // Only run proxy on relevant paths
diff --git a/docs-site/server/content-negotiation.ts b/docs-site/server/content-negotiation.ts
@@ -0,0 +1,105 @@
+const DEFAULT_SUPPORTED_TYPES = [
+  "text/html",
+  "text/markdown",
+  "text/plain",
+  "application/json",
+] as const;
+
+type SupportedType = (typeof DEFAULT_SUPPORTED_TYPES)[number];
+
+type AcceptedType = {
+  type: string;
+  q: number;
+  index: number;
+};
+
+function parseAccept(acceptHeader: string | null): AcceptedType[] {
+  if (!acceptHeader) {
+    return [];
+  }
+
+  return acceptHeader
+    .split(",")
+    .map((part, index) => {
+      const [rawType, ...params] = part.trim().split(";");
+      const type = rawType.toLowerCase();
+      let q = 1;
+
+      for (const param of params) {
+        const [key, value] = param.trim().split("=");
+        if (key === "q") {
+          const parsed = Number.parseFloat(value);
+          q = Number.isFinite(parsed) ? parsed : 0;
+        }
+      }
+
+      return { type, q, index };
+    })
+    .filter(({ type, q }) => Boolean(type) && q > 0);
+}
+
+function matchSpecificity(accepted: string, supported: string): number {
+  if (accepted === supported) {
+    return 2;
+  }
+
+  const [acceptedType, acceptedSubtype] = accepted.split("/");
+  const [supportedType] = supported.split("/");
+
+  if (accepted === "*/*") {
+    return 0;
+  }
+
+  if (acceptedSubtype === "*" && acceptedType === supportedType) {
+    return 1;
+  }
+
+  return -1;
+}
+
+export function negotiateContentType(
+  acceptHeader: string | null,
+  supportedTypes: readonly SupportedType[] = DEFAULT_SUPPORTED_TYPES
+): SupportedType {
+  const acceptedTypes = parseAccept(acceptHeader);
+
+  if (acceptedTypes.length === 0) {
+    return "text/html";
+  }
+
+  let best: {
+    type: SupportedType;
+    q: number;
+    specificity: number;
+    index: number;
+  } | null = null;
+
+  for (const accepted of acceptedTypes) {
+    for (const supported of supportedTypes) {
+      const specificity = matchSpecificity(accepted.type, supported);
+      if (specificity < 0) {
+        continue;
+      }
+
+      const candidate = {
+        type: supported,
+        q: accepted.q,
+        specificity,
+        index: accepted.index,
+      };
+
+      if (
+        !best ||
+        candidate.q > best.q ||
+        (candidate.q === best.q && candidate.specificity > best.specificity) ||
+        (candidate.q === best.q &&
+          candidate.specificity === best.specificity &&
+          candidate.index < best.index)
+      ) {
+        best = candidate;
+      }
+    }
+  }
+
+  return best?.type ?? "text/html";
+}
