feat(website): add llms.txt support for LLM-friendly content

Add support for generating llms.txt and .llms.md files for Quarto websites,
providing LLM-friendly markdown versions of HTML pages.

Features:
- New `llms-txt: true` option in website config
- Generates .llms.md companion files alongside HTML output
- Creates llms.txt index file linking to all markdown pages
- Converts HTML to clean markdown using Pandoc with Lua filter
- Handles callouts (blockquotes with bold type markers)
- Converts images to markdown syntax
- Converts internal links from .html to .llms.md
- Respects draft settings (excludes drafts from output)
- Cleans listing pages (removes empty links, category badges)
- Matches sitemap behavior for incremental builds

New files:
- src/project/types/website/website-llms.ts
- src/resources/filters/llms/llms.lua

Test coverage:
- Basic file generation
- Content conversion (callouts, code, tables, links)
- Draft handling
- Listing page cleanup

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: cscheid

src/project/types/website/website-config.ts

@@ -77,6 +77,7 @@ type WebsiteConfigKey =
   | "other-links"
   | "code-links"
   | "reader-mode"
+  | "llms-txt"
   | "announcement"
   | "draft-mode"
   | "drafts";

src/project/types/website/website-constants.ts

@@ -18,6 +18,7 @@ export const kSiteRepoLinkRel = "repo-link-rel";
 export const kSiteIssueUrl = "issue-url";
 export const kSiteRepoActions = "repo-actions";
 export const kSiteReaderMode = "reader-mode";
+export const kLlmsTxt = "llms-txt";
 
 export const kSiteNavbar = "navbar";
 export const kSiteSidebar = "sidebar";

src/project/types/website/website-llms.ts

@@ -0,0 +1,267 @@
+/*
+ * website-llms.ts
+ *
+ * Copyright (C) 2020-2024 Posit Software, PBC
+ */
+
+import { basename, dirname, join, relative } from "../../../deno_ral/path.ts";
+import { existsSync } from "../../../deno_ral/fs.ts";
+
+import { Document, Element } from "../../../core/deno-dom.ts";
+import { execProcess } from "../../../core/process.ts";
+import { dirAndStem, pathWithForwardSlashes } from "../../../core/path.ts";
+import { pandocBinaryPath, resourcePath } from "../../../core/resources.ts";
+
+import { kProject404File, ProjectContext } from "../../types.ts";
+import { projectOutputDir } from "../../project-shared.ts";
+import { ProjectOutputFile } from "../types.ts";
+
+import { kLlmsTxt } from "./website-constants.ts";
+import {
+  websiteConfigBoolean,
+  websiteDescription,
+  websiteTitle,
+} from "./website-config.ts";
+import { isDraftVisible, isProjectDraft, projectDraftMode } from "./website-utils.ts";
+import { resolveInputTargetForOutputFile } from "../../project-index.ts";
+import { Format } from "../../../config/types.ts";
+
+/**
+ * Compute the output HTML file path from the source file.
+ */
+function computeOutputFilePath(source: string, project: ProjectContext): string {
+  const outputDir = projectOutputDir(project);
+  const sourceRelative = relative(project.dir, source);
+  const [hrefDir, hrefStem] = dirAndStem(sourceRelative);
+  const htmlPath = join(hrefDir, `${hrefStem}.html`);
+  return join(outputDir, htmlPath);
+}
+
+/**
+ * HTML finalizer that generates .llms.md files from rendered HTML.
+ * This runs after all HTML postprocessors have completed.
+ */
+export function llmsHtmlFinalizer(
+  source: string,
+  project: ProjectContext,
+  _format: Format,
+) {
+  return async (doc: Document): Promise<void> => {
+    // Check if llms-txt is enabled
+    if (!websiteConfigBoolean(kLlmsTxt, false, project.config)) {
+      return;
+    }
+
+    // Check draft status via multiple mechanisms
+    const draftMode = projectDraftMode(project);
+
+    // Check 1: quarto:status meta tag (set by draft processing)
+    const statusEl = doc.querySelector("meta[name='quarto:status']");
+    const status = statusEl?.getAttribute("content");
+    const isDraftByStatus = status === "draft" || status === "draft-remove";
+
+    // Check 2: drafts array in project config
+    const sourceRelative = relative(project.dir, source);
+    const isDraftByConfig = isProjectDraft(sourceRelative, project);
+
+    const isDraft = isDraftByStatus || isDraftByConfig;
+
+    if (isDraft && !isDraftVisible(draftMode)) {
+      return; // Skip draft pages
+    }
+
+    // Extract main content from HTML
+    const htmlContent = extractMainContent(doc);
+
+    // Compute the output file path and derive the .llms.md path
+    const outputFile = computeOutputFilePath(source, project);
+    const llmsOutputPath = outputFile.replace(/\.html$/, ".llms.md");
+
+    // Convert HTML to markdown using Pandoc with the llms.lua filter
+    await convertHtmlToLlmsMarkdown(htmlContent, llmsOutputPath);
+  };
+}
+
+/**
+ * Extract the main content from an HTML document, removing navigation,
+ * sidebars, footers, scripts, and styles.
+ */
+function extractMainContent(doc: Document): string {
+  // Clone the document to avoid mutating the original
+  const clone = doc.cloneNode(true) as Document;
+
+  // Remove elements that shouldn't be in llms output
+  const selectorsToRemove = [
+    "#quarto-header",
+    ".nav-footer",
+    "#quarto-sidebar",
+    "#quarto-margin-sidebar",
+    "#quarto-search-results",
+    ".sidebar",
+    ".quarto-search",
+    "nav.navbar",
+    "script",
+    "style",
+    "link[rel='stylesheet']",
+    "meta",
+    "noscript",
+  ];
+
+  for (const selector of selectorsToRemove) {
+    const elements = clone.querySelectorAll(selector);
+    for (const el of elements) {
+      (el as Element).remove();
+    }
+  }
+
+  // Get the main content area
+  const main = clone.querySelector("main") ||
+    clone.querySelector("#quarto-document-content") ||
+    clone.querySelector("article") ||
+    clone.body;
+
+  if (!main) {
+    return "";
+  }
+
+  // Return a minimal HTML document with just the content
+  return `<!DOCTYPE html>
+<html>
+<head><meta charset="utf-8"></head>
+<body>
+${main.innerHTML}
+</body>
+</html>`;
+}
+
+/**
+ * Convert HTML content to markdown using Pandoc with the llms.lua filter.
+ */
+async function convertHtmlToLlmsMarkdown(
+  htmlContent: string,
+  outputPath: string,
+): Promise<void> {
+  const filterPath = resourcePath("filters/llms/llms.lua");
+
+  // Create a temporary file for the HTML content
+  const tempDir = Deno.makeTempDirSync();
+  const tempHtml = join(tempDir, "input.html");
+  Deno.writeTextFileSync(tempHtml, htmlContent);
+
+  try {
+    // Run Pandoc to convert HTML to markdown
+    // Use gfm-raw_html for clean markdown output:
+    // - gfm gives us proper table and code block handling
+    // - -raw_html strips remaining HTML tags, converting figures to markdown images
+    // Note: We use plain "html" input format (not html-native_divs-native_spans)
+    // because native_divs interferes with the Lua filter's callout processing
+    const cmd = [pandocBinaryPath()];
+    cmd.push(tempHtml);
+    cmd.push("-f", "html");
+    cmd.push("-t", "gfm-raw_html");
+    cmd.push("--lua-filter", filterPath);
+    cmd.push("-o", outputPath);
+    cmd.push("--wrap=none");
+
+    const result = await execProcess({
+      cmd: cmd[0],
+      args: cmd.slice(1),
+      stdout: "piped",
+      stderr: "piped",
+    });
+
+    if (!result.success) {
+      console.error(`Error converting HTML to markdown: ${result.stderr}`);
+    }
+  } finally {
+    // Cleanup temp files
+    try {
+      Deno.removeSync(tempDir, { recursive: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}
+
+/**
+ * Generate the llms.txt index file after all HTML files have been rendered.
+ * This is called during the post-render phase.
+ */
+export async function updateLlmsTxt(
+  context: ProjectContext,
+  outputFiles: ProjectOutputFile[],
+  incremental: boolean,
+): Promise<void> {
+  // Check if llms-txt is enabled
+  if (!websiteConfigBoolean(kLlmsTxt, false, context.config)) {
+    return;
+  }
+
+  const outputDir = projectOutputDir(context);
+  const llmsTxtPath = join(outputDir, "llms.txt");
+
+  // Match sitemap behavior: only regenerate on full render
+  if (incremental && existsSync(llmsTxtPath)) {
+    return;
+  }
+
+  const siteTitle = websiteTitle(context.config) || "Untitled";
+  const siteDesc = websiteDescription(context.config) || "";
+  const draftMode = projectDraftMode(context);
+
+  // Helper to check if output file is a draft
+  const isDraft = async (outputFile: ProjectOutputFile): Promise<boolean> => {
+    const index = await resolveInputTargetForOutputFile(
+      context,
+      relative(outputDir, outputFile.file),
+    );
+    return index?.draft ?? false;
+  };
+
+  // Filter out 404 page
+  const doc404 = join(outputDir, kProject404File);
+
+  // Collect all .llms.md files, excluding drafts and 404
+  const llmsFiles: Array<{ path: string; title: string }> = [];
+
+  for (const file of outputFiles) {
+    // Skip 404 page
+    if (file.file === doc404) {
+      continue;
+    }
+
+    // Skip drafts unless in visible draft mode
+    const draft = await isDraft(file);
+    if (draft && !isDraftVisible(draftMode)) {
+      continue;
+    }
+
+    const llmsPath = file.file.replace(/\.html$/, ".llms.md");
+    if (existsSync(llmsPath)) {
+      // Extract title from the format metadata or use filename
+      const title = (file.format.metadata?.title as string) ||
+        basename(file.file, ".html");
+      llmsFiles.push({
+        path: relative(outputDir, llmsPath),
+        title,
+      });
+    }
+  }
+
+  // Generate llms.txt content
+  const lines: string[] = [];
+  lines.push(`# ${siteTitle}`);
+  lines.push("");
+  if (siteDesc) {
+    lines.push(`> ${siteDesc}`);
+    lines.push("");
+  }
+  lines.push("## Pages");
+  lines.push("");
+  for (const f of llmsFiles) {
+    lines.push(`- [${f.title}](${f.path})`);
+  }
+  lines.push("");
+
+  Deno.writeTextFileSync(llmsTxtPath, lines.join("\n"));
+}

src/project/types/website/website.ts

@@ -51,11 +51,13 @@ import { updateSearchIndex } from "./website-search.ts";
 import {
   kDraftMode,
   kDrafts,
+  kLlmsTxt,
   kSiteFavicon,
   kWebsite,
 } from "./website-constants.ts";
 import {
   websiteConfigArray,
+  websiteConfigBoolean,
   websiteConfigString,
   websiteMetadataFields,
   websiteProjectConfig,
@@ -86,6 +88,7 @@ import { formatDate } from "../../../core/date.ts";
 import { projectExtensionPathResolver } from "../../../extension/extension.ts";
 import { websiteDraftPostProcessor } from "./website-draft.ts";
 import { projectDraftMode } from "./website-utils.ts";
+import { llmsHtmlFinalizer, updateLlmsTxt } from "./website-llms.ts";
 import { kFieldCategories } from "./listing/website-listing-shared.ts";
 import { pandocNativeStr } from "../../../core/pandoc/codegen.ts";
 import { asArray } from "../../../core/array.ts";
@@ -353,6 +356,13 @@ export const websiteProjectType: ProjectType = {
       extras.html[kHtmlPostprocessors].push(cookieDep.htmlPostProcessor);
     }
 
+    // Add llms.txt finalizer if enabled
+    if (websiteConfigBoolean(kLlmsTxt, false, project.config)) {
+      extras.html[kHtmlFinalizers]?.push(
+        llmsHtmlFinalizer(source, project, format),
+      );
+    }
+
     return Promise.resolve(extras);
   },
 
@@ -425,6 +435,9 @@ export async function websitePostRender(
   // generate any page aliases
   await updateAliases(context, outputFiles, incremental);
 
+  // generate llms.txt index
+  await updateLlmsTxt(context, outputFiles, incremental);
+
   // write redirecting index.html if there is none
   await ensureIndexPage(context);
 }