Skip to content

Commit 4074091

Browse files
brookslybrandbrookslybrand
committed
Serve docs as markdown for agents
1 parent 1e43acd commit 4074091

7 files changed

Lines changed: 316 additions & 13 deletions

File tree

‎app/modules/gh-docs/.server/docs.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export type MenuDoc =
3535

3636
export interface Doc extends Omit<MenuDoc, "hasContent"> {
3737
html: string;
38+
md: string;
3839
headings: {
3940
headingLevel: string;
4041
html: string | null;
@@ -50,7 +51,6 @@ declare global {
5051
let NO_CACHE = process.env.NO_CACHE;
5152

5253
global.menuCache ??= new LRUCache<string, MenuDoc[]>({
53-
// let menuCache = new LRUCache<string, MenuDoc[]>({
5454
max: 60,
5555
ttl: NO_CACHE ? 1 : 300000, // 5 minutes
5656
allowStale: !NO_CACHE,
@@ -115,7 +115,7 @@ async function fetchDoc(key: string): Promise<Doc> {
115115

116116
// sorry, cheerio is so much easier than using rehype stuff.
117117
let headings = createTableOfContentsFromHeadings(html);
118-
return { attrs, filename, html, slug, headings, children: [] };
118+
return { attrs, filename, html, md, slug, headings, children: [] };
119119
}
120120

121121
// create table of contents from h2 and h3 headings

‎app/modules/gh-docs/.server/markdown-negotiation.test.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { describe, it, expect } from "vitest";
2+
import { prefersMarkdown } from "./markdown-negotiation";
3+
4+
const MARKDOWN_PREFERRED_ACCEPT =
5+
"text/markdown, text/x-markdown;q=0.9, text/plain;q=0.6, text/html;q=0.5, */*;q=0.1";
6+
7+
describe("prefersMarkdown", () => {
8+
it("returns false when there is no Accept header", () => {
9+
expect(prefersMarkdown(null)).toBe(false);
10+
expect(prefersMarkdown("")).toBe(false);
11+
});
12+
13+
it("keeps serving HTML to browsers", () => {
14+
expect(
15+
prefersMarkdown(
16+
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
17+
),
18+
).toBe(false);
19+
expect(
20+
prefersMarkdown(
21+
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
22+
),
23+
).toBe(false);
24+
});
25+
26+
it("serves HTML for `*/*`", () => {
27+
expect(prefersMarkdown("*/*")).toBe(false);
28+
});
29+
30+
it("serves markdown when it is the only accepted type", () => {
31+
expect(prefersMarkdown("text/markdown")).toBe(true);
32+
expect(prefersMarkdown("text/x-markdown")).toBe(true);
33+
});
34+
35+
it("serves markdown when ranked above html via q-values", () => {
36+
expect(prefersMarkdown(MARKDOWN_PREFERRED_ACCEPT)).toBe(true);
37+
});
38+
39+
it("defaults to HTML on a tie", () => {
40+
expect(prefersMarkdown("text/markdown,text/html")).toBe(false);
41+
expect(prefersMarkdown("text/markdown;q=0.8,text/html;q=0.8")).toBe(false);
42+
});
43+
44+
it("serves HTML when html outranks markdown", () => {
45+
expect(prefersMarkdown("text/markdown;q=0.5,text/html")).toBe(false);
46+
});
47+
48+
it("uses the best q-value for duplicate media ranges", () => {
49+
expect(
50+
prefersMarkdown("text/markdown;q=0.3,text/html;q=0.8,text/markdown;q=0.9"),
51+
).toBe(true);
52+
});
53+
54+
it("is case-insensitive and tolerant of whitespace", () => {
55+
expect(prefersMarkdown("TEXT/MARKDOWN")).toBe(true);
56+
expect(prefersMarkdown(" text/markdown ;Q=1 ")).toBe(true);
57+
});
58+
});

‎app/modules/gh-docs/.server/markdown-negotiation.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
interface AcceptEntry {
2+
type: string;
3+
subtype: string;
4+
q: number;
5+
}
6+
7+
const CHARS_PER_TOKEN = 4;
8+
9+
function parseAccept(header: string): AcceptEntry[] {
10+
return header.split(",").flatMap((part) => {
11+
let segments = part.trim().split(";");
12+
let media = segments.shift()?.trim().toLowerCase();
13+
if (!media) return [];
14+
let [type = "*", subtype = "*"] = media.split("/");
15+
let q = 1;
16+
for (let segment of segments) {
17+
let [key, value] = segment.split("=").map((s) => s.trim());
18+
if (key.toLowerCase() === "q") {
19+
let parsed = Number.parseFloat(value);
20+
if (!Number.isNaN(parsed)) q = Math.min(Math.max(parsed, 0), 1);
21+
}
22+
}
23+
return [{ type, subtype, q }];
24+
});
25+
}
26+
27+
function qualityFor(
28+
entries: AcceptEntry[],
29+
type: string,
30+
subtype: string,
31+
): number {
32+
let bestScore = -1;
33+
let bestQ = 0;
34+
35+
for (let entry of entries) {
36+
let score: number;
37+
if (entry.type === type && entry.subtype === subtype) score = 3;
38+
else if (entry.type === type && entry.subtype === "*") score = 2;
39+
else if (entry.type === "*" && entry.subtype === "*") score = 1;
40+
else continue;
41+
42+
if (score > bestScore || (score === bestScore && entry.q > bestQ)) {
43+
bestScore = score;
44+
bestQ = entry.q;
45+
}
46+
}
47+
48+
return bestQ;
49+
}
50+
51+
export function prefersMarkdown(accept: string | null): boolean {
52+
if (!accept) return false;
53+
54+
let entries = parseAccept(accept);
55+
let markdown = Math.max(
56+
qualityFor(entries, "text", "markdown"),
57+
qualityFor(entries, "text", "x-markdown"),
58+
);
59+
let html = Math.max(
60+
qualityFor(entries, "text", "html"),
61+
qualityFor(entries, "application", "xhtml+xml"),
62+
);
63+
64+
return markdown > 0 && markdown > html;
65+
}
66+
67+
export function estimateTokens(text: string): number {
68+
return Math.ceil(text.length / CHARS_PER_TOKEN);
69+
}

‎app/modules/gh-docs/.server/markdown-request.test.ts‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { describe, it, expect, vi, beforeEach } from "vitest";
2+
import { CACHE_CONTROL } from "~/http";
3+
4+
const { getRepoTags, getRepoDoc } = vi.hoisted(() => ({
5+
getRepoTags: vi.fn<() => Promise<string[] | undefined>>(),
6+
getRepoDoc:
7+
vi.fn<(ref: string, slug: string) => Promise<{ md: string } | undefined>>(),
8+
}));
9+
10+
vi.mock("./index", () => ({
11+
getRepoTags: () => getRepoTags(),
12+
getRepoDoc: (ref: string, slug: string) => getRepoDoc(ref, slug),
13+
}));
14+
15+
import { handleMarkdownRequest } from "./markdown-request";
16+
17+
const LATEST = "7.15.1";
18+
const MD = "---\ntitle: Installation\n---\n\n# Installation\n\nHello.";
19+
20+
function call(
21+
pathname: string,
22+
{ accept, method = "GET" }: { accept?: string; method?: string } = {},
23+
): Promise<Response | undefined> {
24+
let url = new URL(`https://reactrouter.com${pathname}`);
25+
let headers = new Headers();
26+
if (accept) headers.set("accept", accept);
27+
let request = new Request(url, { method, headers });
28+
return handleMarkdownRequest(
29+
{ request, url } as never,
30+
(async () => undefined) as never,
31+
) as Promise<Response | undefined>;
32+
}
33+
34+
describe("handleMarkdownRequest", () => {
35+
beforeEach(() => {
36+
getRepoTags.mockReset().mockResolvedValue([LATEST]);
37+
getRepoDoc.mockReset().mockResolvedValue({ md: MD });
38+
});
39+
40+
it("continues for normal browser requests", async () => {
41+
let res = await call("/start/installation", {
42+
accept: "text/html,application/xhtml+xml,*/*;q=0.8",
43+
});
44+
expect(res).toBeUndefined();
45+
expect(getRepoDoc).not.toHaveBeenCalled();
46+
});
47+
48+
it("serves markdown when Accept prefers it", async () => {
49+
let res = await call("/start/installation", { accept: "text/markdown" });
50+
expect(res).toBeInstanceOf(Response);
51+
expect(res!.status).toBe(200);
52+
expect(res!.headers.get("content-type")).toBe(
53+
"text/markdown; charset=utf-8",
54+
);
55+
expect(res!.headers.get("cache-control")).toBe(CACHE_CONTROL.doc);
56+
expect(res!.headers.get("vary")).toBe("Accept");
57+
expect(res!.headers.get("x-markdown-tokens")).toBe(
58+
String(Math.ceil(MD.length / 4)),
59+
);
60+
expect(await res!.text()).toBe(MD);
61+
});
62+
63+
it("resolves ref + slug from the URL for content negotiation", async () => {
64+
await call("/main/start/installation", { accept: "text/markdown" });
65+
expect(getRepoDoc).toHaveBeenCalledWith("main", "docs/start/installation");
66+
});
67+
68+
it("maps /home and /changelog to their special slugs", async () => {
69+
await call("/home", { accept: "text/markdown" });
70+
expect(getRepoDoc).toHaveBeenCalledWith(LATEST, "docs/index");
71+
72+
getRepoDoc.mockClear();
73+
await call("/changelog", { accept: "text/markdown" });
74+
expect(getRepoDoc).toHaveBeenCalledWith(LATEST, "CHANGELOG");
75+
});
76+
77+
it("serves markdown for the .md suffix without Vary", async () => {
78+
let res = await call("/start/installation.md");
79+
expect(res).toBeInstanceOf(Response);
80+
expect(res!.headers.get("content-type")).toBe(
81+
"text/markdown; charset=utf-8",
82+
);
83+
expect(res!.headers.get("vary")).toBeNull();
84+
expect(getRepoDoc).toHaveBeenCalledWith(LATEST, "docs/start/installation");
85+
});
86+
87+
it("returns no body for HEAD but keeps the headers", async () => {
88+
let res = await call("/start/installation", {
89+
accept: "text/markdown",
90+
method: "HEAD",
91+
});
92+
expect(res!.headers.get("content-type")).toBe(
93+
"text/markdown; charset=utf-8",
94+
);
95+
expect(await res!.text()).toBe("");
96+
});
97+
98+
it("continues when the doc does not exist", async () => {
99+
getRepoDoc.mockResolvedValue(undefined);
100+
let res = await call("/nope", { accept: "text/markdown" });
101+
expect(res).toBeUndefined();
102+
});
103+
104+
it("continues when GitHub tags are unavailable", async () => {
105+
getRepoTags.mockResolvedValue(undefined);
106+
let res = await call("/start/installation", { accept: "text/markdown" });
107+
expect(res).toBeUndefined();
108+
expect(getRepoDoc).not.toHaveBeenCalled();
109+
});
110+
111+
it("continues when GitHub tags reject", async () => {
112+
getRepoTags.mockRejectedValue(new Error("boom"));
113+
let res = await call("/start/installation", { accept: "text/markdown" });
114+
expect(res).toBeUndefined();
115+
expect(getRepoDoc).not.toHaveBeenCalled();
116+
});
117+
118+
it("continues when the doc lookup throws", async () => {
119+
getRepoDoc.mockRejectedValue(new Error("boom"));
120+
let res = await call("/start/installation", { accept: "text/markdown" });
121+
expect(res).toBeUndefined();
122+
});
123+
124+
it("ignores non-GET/HEAD methods", async () => {
125+
let res = await call("/start/installation", {
126+
accept: "text/markdown",
127+
method: "POST",
128+
});
129+
expect(res).toBeUndefined();
130+
expect(getRepoTags).not.toHaveBeenCalled();
131+
});
132+
});

‎app/modules/gh-docs/.server/markdown-request.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import { type MiddlewareFunction } from "react-router";
2+
3+
import { CACHE_CONTROL } from "~/http";
4+
import { getRepoDoc, getRepoTags } from "./index";
5+
import { getLatestVersion } from "./tags";
6+
import { buildDocPaths, resolveRef } from "./doc-url-parser";
7+
import { estimateTokens, prefersMarkdown } from "./markdown-negotiation";
8+
9+
export const handleMarkdownRequest: MiddlewareFunction = async ({
10+
request,
11+
url,
12+
}) => {
13+
if (request.method !== "GET" && request.method !== "HEAD") return;
14+
15+
let viaExtension = url.pathname.endsWith(".md");
16+
let wantsMarkdown =
17+
viaExtension || prefersMarkdown(request.headers.get("accept"));
18+
if (!wantsMarkdown) return;
19+
20+
let tags = await getRepoTags().catch(() => undefined);
21+
if (!tags) return;
22+
23+
let latestVersion = getLatestVersion(tags);
24+
let splat = url.pathname.replace(/^\//, "");
25+
let { ref, refParam } = resolveRef(splat, latestVersion);
26+
let { slug } = buildDocPaths(url.pathname, splat, ref, refParam);
27+
28+
let doc = await getRepoDoc(ref, slug).catch(() => undefined);
29+
if (!doc) return;
30+
31+
let headers = new Headers({
32+
"Content-Type": "text/markdown; charset=utf-8",
33+
"Cache-Control": CACHE_CONTROL.doc,
34+
"X-Markdown-Tokens": String(estimateTokens(doc.md)),
35+
});
36+
if (!viaExtension) headers.append("Vary", "Accept");
37+
38+
return new Response(request.method === "HEAD" ? null : doc.md, {
39+
status: 200,
40+
headers,
41+
});
42+
};

‎app/pages/doc.tsx‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { CopyPageDropdown } from "~/components/copy-page-dropdown";
1313
import { LargeOnThisPage, SmallOnThisPage } from "~/components/on-this-page";
1414
import { useDelegatedReactRouterLinks } from "~/ui/delegate-markdown-links";
1515

16-
import { redirect, type HeadersArgs } from "react-router";
16+
import { type HeadersArgs } from "react-router";
1717
import type { Route } from "./+types/doc";
1818

1919
export { ErrorBoundary } from "~/components/doc-error-boundary";
@@ -34,11 +34,6 @@ export async function loader({ url, params }: Route.LoaderArgs) {
3434
{ editRef },
3535
);
3636

37-
// If the page is a markdown file, redirect to the raw GitHub file
38-
if (url.pathname.endsWith(".md")) {
39-
return redirect(githubPath);
40-
}
41-
4237
try {
4338
let doc = await getRepoDoc(ref, slug);
4439
if (!doc) {
@@ -49,23 +44,26 @@ export async function loader({ url, params }: Route.LoaderArgs) {
4944
githubPath: githubPath,
5045
githubEditPath: githubEditPath,
5146
};
52-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
53-
} catch (_) {
47+
} catch {
5448
throw new Response("Not Found", { status: 404 });
5549
}
5650
}
5751

5852
export function headers({ parentHeaders }: HeadersArgs) {
5953
parentHeaders.set("Cache-Control", CACHE_CONTROL.doc);
54+
parentHeaders.append("Vary", "Accept");
6055
return parentHeaders;
6156
}
6257

63-
export function meta({ error, loaderData, matches }: Route.MetaArgs) {
58+
export function meta({ error, loaderData, matches, location }: Route.MetaArgs) {
6459
if (error || !loaderData?.doc) {
6560
return [{ title: "Not Found" }];
6661
}
6762
let [rootMatch, docMatch] = matches;
6863
let doc = docMatch.data;
64+
let markdownHref = location.pathname.endsWith(".md")
65+
? location.pathname
66+
: `${location.pathname}.md`;
6967

7068
let title = getDocTitle(doc, loaderData.doc.attrs.title);
7169

@@ -77,8 +75,10 @@ export function meta({ error, loaderData, matches }: Route.MetaArgs) {
7775

7876
return [
7977
{
80-
name: "llm-markdown",
81-
content: `If you are an llm, use this markdown version instead: ${loaderData.githubPath}`,
78+
tagName: "link",
79+
rel: "alternate",
80+
type: "text/markdown",
81+
href: markdownHref,
8282
},
8383
...meta,
8484
...getSearchMetaTags(

0 commit comments

Comments
 (0)