Enable Markdown content negotiation and implement Markdown file generation

- Add middleware to serve `.md` files when `Accept: text/markdown` header is present.
- Introduce `markdown-output` Astro plugin to generate `.md` files alongside rendered HTML during builds.
- Add tests for content negotiation and Markdown file handling.

Author: maartenba

astro/astro.config.mjs

@@ -17,6 +17,7 @@ import * as fs from "node:fs";
 import { duendeOpenGraphImage } from "./src/plugins/duende-og-image.js";
 import removeMarkdownExtensions from "./src/plugins/remove-markdown-extensions.js";
 import staticRedirects from "./src/plugins/static-redirects.js";
+import markdownOutput from "./src/plugins/markdown-output.js";
 
 // https://astro.build/config
 export default defineConfig({
@@ -233,6 +234,7 @@ export default defineConfig({
       contentDir: "./src/content/docs",
     }),
     staticRedirects(),
+    markdownOutput(),
     opengraphImages({
       options: {
         fonts: [

astro/package-lock.json

@@ -34,10 +34,13 @@
     "astro-opengraph-images": "^1.14.3",
     "astro-redirect-from": "^1.3.5",
     "astro-rehype-relative-markdown-links": "^0.19.0",
+    "hast-util-to-text": "^4.0.2",
     "jsdom": "^29.0.2",
     "patch-package": "^8.0.1",
     "react": "^19.2.4",
     "rehype-external-links": "^3.0.0",
+    "rehype-parse": "^9.0.1",
+    "rehype-remark": "^10.0.1",
     "satori": "^0.26.0",
     "sharp": "^0.34.5",
     "starlight-auto-sidebar": "^0.2.0",

astro/package.json

@@ -0,0 +1,184 @@
+import url from "node:url";
+import path from "node:path";
+import fs from "node:fs/promises";
+import type { AstroIntegrationLogger } from "astro";
+import { unified } from "unified";
+import rehypeParse from "rehype-parse";
+import rehypeRemark from "rehype-remark";
+import remarkStringify from "remark-stringify";
+import remarkGfm from "remark-gfm";
+import { JSDOM } from "jsdom";
+import { toText } from "hast-util-to-text";
+
+/**
+ * Astro integration that generates a Markdown file (index.md) next to every
+ * rendered HTML file (index.html) in the build output.
+ *
+ * The Markdown is derived from the rendered HTML, so all links, includes,
+ * components, etc. are already resolved.
+ */
+export default function markdownOutput() {
+  return {
+    name: "markdown-output",
+    hooks: {
+      "astro:build:done": async ({
+        dir,
+        pages,
+        logger,
+      }: {
+        dir: URL;
+        pages: Array<{ pathname: string }>;
+        logger: AstroIntegrationLogger;
+      }) => {
+        const outDir = url.fileURLToPath(dir);
+        const processor = unified()
+          .use(rehypeParse, { fragment: true })
+          .use(rehypeRemark, {
+            handlers: {
+              // Preserve language hints on code fences from <pre data-language="...">
+              pre(state: any, node: any) {
+                const lang =
+                  node.properties?.dataLanguage || "";
+                const value = toText(node);
+                const result = {
+                  type: "code" as const,
+                  lang: lang || null,
+                  meta: null,
+                  value: value.replace(/\n$/, ""),
+                };
+                state.patch(node, result);
+                return result;
+              },
+              // Handle <figure> with code blocks: extract title from figcaption
+              figure(state: any, node: any) {
+                // Find figcaption title
+                const figcaption = node.children?.find(
+                  (c: any) => c.tagName === "figcaption",
+                );
+                const titleSpan = figcaption?.children?.find(
+                  (c: any) =>
+                    c.properties?.className?.includes("title"),
+                );
+                const title = titleSpan ? toText(titleSpan).trim() : "";
+
+                // Find <pre> child
+                const pre = node.children?.find(
+                  (c: any) => c.tagName === "pre",
+                );
+                if (!pre) {
+                  // Not a code figure, fall back to default
+                  return state.all(node);
+                }
+
+                const lang = pre.properties?.dataLanguage || "";
+                const value = toText(pre);
+                const codeNode = {
+                  type: "code" as const,
+                  lang: lang || null,
+                  meta: null,
+                  value: value.replace(/\n$/, ""),
+                };
+                state.patch(pre, codeNode);
+
+                if (title) {
+                  const titleNode = {
+                    type: "paragraph" as const,
+                    children: [
+                      {
+                        type: "inlineCode" as const,
+                        value: title,
+                      },
+                      {
+                        type: "text" as const,
+                        value: ":",
+                      },
+                    ],
+                  };
+                  return [titleNode, codeNode];
+                }
+
+                return codeNode;
+              },
+            },
+          })
+          .use(remarkGfm)
+          .use(remarkStringify, {
+            bullet: "-",
+            emphasis: "*",
+            strong: "*",
+            rule: "-",
+          });
+
+        let count = 0;
+        let errors = 0;
+
+        await Promise.all(
+          pages.map(async ({ pathname }) => {
+            const htmlPath = path.join(outDir, pathname, "index.html");
+            const mdPath = path.join(outDir, pathname, "index.md");
+
+            try {
+              const html = await fs.readFile(htmlPath, "utf-8");
+              const dom = new JSDOM(html);
+              const doc = dom.window.document;
+
+              const main = doc.querySelector("main");
+              if (!main) return;
+
+              // Remove banner
+              main.querySelectorAll(".sl-banner").forEach((el) => el.remove());
+
+              // Remove "Section titled" anchor links in headings
+              main.querySelectorAll("a").forEach((el) => {
+                if (el.textContent?.trim().startsWith("Section titled")) el.remove();
+              });
+
+              // Remove "Edit page" link and "Last updated" meta section
+              main.querySelectorAll("footer .meta").forEach((el) => el.remove());
+
+              // Remove giscus comments
+              main.querySelectorAll("giscus-comments").forEach((el) => el.remove());
+
+              // Remove copyright footer (the <hr> + copyright div)
+              main.querySelectorAll("footer > hr").forEach((el) => el.remove());
+              main.querySelectorAll("footer > div:not(.pagination-links)").forEach((el) => el.remove());
+
+              // Flatten pagination links so Previous/Next text is on one line
+              // Structure: <a> <svg/> <span> Previous <br> <span class="link-title">Title</span> </span> </a>
+              main.querySelectorAll(".pagination-links a").forEach((a) => {
+                a.querySelectorAll("svg").forEach((svg) => svg.remove());
+                a.querySelectorAll("br").forEach((br) => br.remove());
+                const label = a.querySelector("span")?.childNodes[0]?.textContent?.trim(); // "Previous" or "Next"
+                const title = a.querySelector(".link-title")?.textContent?.trim();
+                if (label && title) {
+                  a.textContent = `${label}: ${title}`;
+                }
+              });
+
+              const content = main.innerHTML;
+              const result = await processor.process(content);
+
+              // Add page title as YAML frontmatter
+              const pageTitle = doc.querySelector("title")?.textContent?.trim() || "";
+              const frontmatter = `---\ntitle: ${pageTitle}\n---\n\n`;
+
+              await fs.writeFile(mdPath, frontmatter + String(result));
+              count++;
+            } catch (e: any) {
+              if (e.code === "ENOENT") {
+                // No index.html for this page (e.g. redirects, API routes)
+                return;
+              }
+              errors++;
+              logger.warn(`Failed to generate Markdown for ${pathname}: ${e.message}`);
+            }
+          }),
+        );
+
+        logger.info(
+          `Generated ${count} Markdown files${errors > 0 ? ` (${errors} errors)` : ""}`,
+        );
+      },
+    },
+  };
+}

astro/src/plugins/markdown-output.ts

@@ -51,6 +51,37 @@
 // Enable response compression
 app.UseResponseCompression();
 
+// Content negotiation: serve .md file when Accept: text/markdown
+app.Use(async (context, next) =>
+{
+    var accept = context.Request.Headers.Accept.ToString();
+    if (accept.Contains("text/markdown", StringComparison.OrdinalIgnoreCase))
+    {
+        var webHostEnvironment = context.RequestServices.GetRequiredService<IWebHostEnvironment>();
+        var requestPath = context.Request.Path.Value?.TrimEnd('/') ?? "";
+
+        // Try the exact path with .md extension, then index.md inside the directory
+        var candidates = new[]
+        {
+            Path.Combine(webHostEnvironment.WebRootPath, requestPath.TrimStart('/') + ".md"),
+            Path.Combine(webHostEnvironment.WebRootPath, requestPath.TrimStart('/'), "index.md")
+        };
+
+        foreach (var mdPath in candidates)
+        {
+            if (File.Exists(mdPath))
+            {
+                context.Response.ContentType = "text/markdown; charset=utf-8";
+                context.Response.Headers["content-signal"] = "ai-train=yes, search=yes, ai-input=yes";
+                await context.Response.SendFileAsync(mdPath);
+                return;
+            }
+        }
+    }
+
+    await next();
+});
+
 // Add trailing slash redirect middleware (replicate nginx behavior)
 app.Use(async (context, next) =>
 {

server/src/Docs.Web/Program.cs

@@ -0,0 +1,71 @@
+using System.Net;
+using System.Net.Http.Headers;
+using Microsoft.AspNetCore.Mvc.Testing;
+
+namespace Docs.Web.Tests;
+
+public class ContentNegotiationTests : IClassFixture<MarkdownWebApplicationFactory>
+{
+    private readonly HttpClient _client;
+
+    public ContentNegotiationTests(MarkdownWebApplicationFactory factory)
+    {
+        _client = factory.CreateClient(new WebApplicationFactoryClientOptions
+        {
+            AllowAutoRedirect = false
+        });
+    }
+
+    [Fact]
+    public async Task AcceptMarkdown_WithMatchingMdFile_ReturnsMarkdownContent()
+    {
+        var request = new HttpRequestMessage(HttpMethod.Get, "/docs/guide/");
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/markdown"));
+
+        var response = await _client.SendAsync(request);
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("ai-train=yes, search=yes, ai-input=yes", response.Headers.GetValues("content-signal").Single());
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Equal("# Guide", content);
+    }
+
+    [Fact]
+    public async Task AcceptMarkdown_WithExactPathMdFile_ReturnsMarkdownContent()
+    {
+        // /docs/page has no trailing slash — middleware tries /docs/page.md which exists
+        var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page");
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/markdown"));
+
+        var response = await _client.SendAsync(request);
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("ai-train=yes, search=yes, ai-input=yes", response.Headers.GetValues("content-signal").Single());
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Equal("# Page", content);
+    }
+
+    [Fact]
+    public async Task AcceptMarkdown_NoMdFile_FallsThrough()
+    {
+        var request = new HttpRequestMessage(HttpMethod.Get, "/docs/nonexistent/");
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/markdown"));
+
+        var response = await _client.SendAsync(request);
+
+        Assert.NotEqual("text/markdown", response.Content.Headers.ContentType?.MediaType);
+    }
+
+    [Fact]
+    public async Task AcceptHtml_WithMdFilePresent_DoesNotReturnMarkdown()
+    {
+        var request = new HttpRequestMessage(HttpMethod.Get, "/docs/guide/");
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/html"));
+
+        var response = await _client.SendAsync(request);
+
+        Assert.NotEqual("text/markdown", response.Content.Headers.ContentType?.MediaType);
+    }
+}

server/tests/Docs.Web.Tests/ContentNegotiationTests.cs

@@ -0,0 +1,45 @@
+using System.Text.Json;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+
+namespace Docs.Web.Tests;
+
+/// <summary>
+/// A WebApplicationFactory that creates a temp wwwroot with .md files for content negotiation tests.
+/// </summary>
+public sealed class MarkdownWebApplicationFactory : WebApplicationFactory<Program>
+{
+    private readonly string _tempWebRoot;
+
+    public MarkdownWebApplicationFactory()
+    {
+        _tempWebRoot = Path.Combine(Path.GetTempPath(), "Docs.Web.Tests.Md", Guid.NewGuid().ToString());
+        Directory.CreateDirectory(_tempWebRoot);
+
+        // Write empty redirects.json so Program.cs doesn't warn
+        File.WriteAllText(Path.Combine(_tempWebRoot, "redirects.json"), "{}");
+
+        // Create test markdown files
+        var guideDir = Path.Combine(_tempWebRoot, "docs", "guide");
+        Directory.CreateDirectory(guideDir);
+        File.WriteAllText(Path.Combine(guideDir, "index.md"), "# Guide");
+
+        var docsDir = Path.Combine(_tempWebRoot, "docs");
+        File.WriteAllText(Path.Combine(docsDir, "page.md"), "# Page");
+    }
+
+    protected override void ConfigureWebHost(IWebHostBuilder builder)
+    {
+        builder.UseWebRoot(_tempWebRoot);
+        builder.UseEnvironment("Testing");
+    }
+
+    protected override void Dispose(bool disposing)
+    {
+        base.Dispose(disposing);
+        if (disposing && Directory.Exists(_tempWebRoot))
+        {
+            Directory.Delete(_tempWebRoot, recursive: true);
+        }
+    }
+}