Merge pull request #525 from DeterminateSystems/lucperkins/gtm-256-ai-readiness

Add robots.txt file and Markdown negotiation

Author: lucperkins

netlify.toml

@@ -12,6 +12,16 @@ NODE_VERSION = "24.13.0" # Keep this in step with `nix develop --command node --
 [context.production]
 environment = { ENV = "production" }
 
+[[edge_functions]]
+function = "markdown"
+path = "/concepts/*"
+excludedPath = ["/concepts/*.md"]
+
+[[edge_functions]]
+function = "markdown"
+path = "/start/*"
+excludedPath = ["/start/*.md"]
+
 [[redirects]]
 from = "/quick-start"
 to = "/start.html"

netlify/edge-functions/markdown.ts

@@ -0,0 +1,70 @@
+// This function provides the desired server dance for .md endpoints
+// that perform Markdown content negotiation for AI agents
+import type { Context } from "@netlify/edge-functions";
+
+function prefersMarkdown(accept: string | null): boolean {
+  if (!accept) return false;
+  let md = 0;
+  let html = 0;
+  for (const part of accept.split(",")) {
+    const [type, ...params] = part
+      .trim()
+      .split(";")
+      .map((s) => s.trim());
+    const qParam = params.find((p) => p.startsWith("q="));
+    const q = qParam ? parseFloat(qParam.slice(2)) : 1;
+    if (Number.isNaN(q)) continue;
+    if (type === "text/markdown") md = Math.max(md, q);
+    else if (type === "text/html" || type === "application/xhtml+xml") {
+      html = Math.max(html, q);
+    }
+  }
+  return md > 0 && md >= html;
+}
+
+export default async (request: Request, context: Context) => {
+  const url = new URL(request.url);
+
+  const looksLikeAsset = /\.[a-z0-9]+$/i.test(url.pathname);
+  if (request.method !== "GET" || looksLikeAsset) {
+    return context.next();
+  }
+
+  const accept = request.headers.get("Accept");
+  const mdPath = url.pathname.replace(/\/$/, "") + ".md";
+  const mdUrl = new URL(mdPath, url);
+
+  if (prefersMarkdown(accept)) {
+    try {
+      const mdResponse = await fetch(mdUrl, {
+        headers: { "User-Agent": request.headers.get("User-Agent") ?? "" },
+      });
+      const upstreamCt = mdResponse.headers.get("Content-Type") ?? "";
+      if (mdResponse.ok && upstreamCt.includes("markdown")) {
+        return new Response(mdResponse.body, {
+          status: 200,
+          headers: {
+            "Content-Type": "text/markdown; charset=utf-8",
+            Vary: "Accept",
+            "Cache-Control": "public, max-age=3600",
+            Link: `<${url.href}>; rel="canonical"`,
+          },
+        });
+      }
+    } catch {
+      // Upstream .md fetch failed; falling through to normal HTML response
+    }
+  }
+
+  // Fell through to normal HTML response due to lack of .md sibling
+  const response = await context.next();
+  const ct = response.headers.get("Content-Type") ?? "";
+  if (ct.includes("text/html")) {
+    response.headers.append(
+      "Link",
+      `<${mdUrl.href}>; rel="alternate"; type="text/markdown"`,
+    );
+    response.headers.append("Vary", "Accept");
+  }
+  return response;
+};

package-lock.json

@@ -25,6 +25,7 @@
     "@iconify-json/fa-solid": "^1.2.2",
     "@iconify-json/heroicons": "^1.2.3",
     "@iconify-json/radix-icons": "^1.2.6",
+    "@netlify/edge-functions": "^3.0.6",
     "@tailwindcss/typography": "^0.5.19",
     "@types/alpinejs": "^3.13.11",
     "@types/react": "^18.3.28",

package.json

@@ -0,0 +1,5 @@
+User-agent: *
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
+Allow: /
+
+Sitemap: https://zero-to-nix.com/sitemap-index.xml

public/robots.txt

@@ -0,0 +1,39 @@
+import type { APIRoute, GetStaticPaths } from "astro";
+import { type CollectionEntry, getCollection } from "astro:content";
+
+export const prerender = true;
+
+type Props = { entry: CollectionEntry<"concepts"> };
+
+const stripExt = (id: string) => id.replace(/\.(md|mdx)$/i, "");
+
+export const getStaticPaths = (async () => {
+  const entries = await getCollection("concepts");
+  return entries.map((entry) => ({
+    params: { slug: stripExt(entry.id) },
+    props: { entry },
+  }));
+}) satisfies GetStaticPaths;
+
+export const GET: APIRoute<Props> = async ({ props, params }) => {
+  let entry: CollectionEntry<"concepts"> | undefined = props?.entry;
+  if (!entry && typeof params.slug === "string") {
+    const all = await getCollection("concepts");
+    entry = all.find((e) => stripExt(e.id) === params.slug);
+  }
+  if (!entry) {
+    return new Response("Not Found", { status: 404 });
+  }
+
+  const { title } = entry.data;
+  const header = [title && `# ${title}`].filter(Boolean).join("\n");
+
+  const body = header ? `${header}\n\n${entry.body}` : entry.body;
+
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "public, max-age=3600",
+    },
+  });
+};

src/pages/concepts/[slug].md.ts

@@ -0,0 +1,39 @@
+import type { APIRoute, GetStaticPaths } from "astro";
+import { type CollectionEntry, getCollection } from "astro:content";
+
+export const prerender = true;
+
+type Props = { entry: CollectionEntry<"start"> };
+
+const stripExt = (id: string) => id.replace(/\.(md|mdx)$/i, "");
+
+export const getStaticPaths = (async () => {
+  const entries = await getCollection("start");
+  return entries.map((entry) => ({
+    params: { slug: stripExt(entry.id) },
+    props: { entry },
+  }));
+}) satisfies GetStaticPaths;
+
+export const GET: APIRoute<Props> = async ({ props, params }) => {
+  let entry: CollectionEntry<"start"> | undefined = props?.entry;
+  if (!entry && typeof params.slug === "string") {
+    const all = await getCollection("start");
+    entry = all.find((e) => stripExt(e.id) === params.slug);
+  }
+  if (!entry) {
+    return new Response("Not Found", { status: 404 });
+  }
+
+  const { title } = entry.data;
+  const header = [title && `# ${title}`].filter(Boolean).join("\n");
+
+  const body = header ? `${header}\n\n${entry.body}` : entry.body;
+
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "public, max-age=3600",
+    },
+  });
+};