refac

Author: tjbck

docs/intro.mdx

@@ -168,3 +168,16 @@ Need **custom branding**, **SLA support**, or **Long-Term Support (LTS)** versio
   </a>
 
 </div>
+
+---
+
+## For LLMs and coding agents
+
+Machine-readable entry points to this documentation:
+
+- [llms.txt](https://docs.openwebui.com/llms.txt): curated index of every docs page with short descriptions.
+- [llms-full.txt](https://docs.openwebui.com/llms-full.txt): every docs page concatenated into a single Markdown file.
+- [agents.txt](https://docs.openwebui.com/agents.txt): concise instructions for agents using these docs.
+- [api/search?q=YOUR_QUERY](https://docs.openwebui.com/api/search?q=YOUR_QUERY): deterministic docs search.
+
+All files are generated fresh on every deploy.

package.json

@@ -6,9 +6,10 @@
 		"docusaurus": "docusaurus",
 		"start": "docusaurus start",
 		"build": "docusaurus build",
-		"postbuild": "node scripts/generate-raw-docs.mjs && node scripts/generate-search-corpus.mjs",
+		"postbuild": "node scripts/generate-raw-docs.mjs && node scripts/generate-search-corpus.mjs && node scripts/generate-llms-files.mjs",
 		"raw-docs": "node scripts/generate-raw-docs.mjs",
 		"search-corpus": "node scripts/generate-search-corpus.mjs",
+		"llms-files": "node scripts/generate-llms-files.mjs",
 		"test:docs-search": "node scripts/test-docs-search.mjs",
 		"swizzle": "docusaurus swizzle",
 		"deploy": "docusaurus deploy",

scripts/generate-llms-files.mjs

@@ -0,0 +1,94 @@
+import fs from "node:fs";
+import path from "node:path";
+
+const BUILD_DIR = "build";
+const CORPUS_PATH = path.join(BUILD_DIR, "search-corpus.json");
+const LLMS_PATH = path.join(BUILD_DIR, "llms.txt");
+const LLMS_FULL_PATH = path.join(BUILD_DIR, "llms-full.txt");
+
+function normalizeWhitespace(text) {
+	return text.replace(/\s+/g, " ").trim();
+}
+
+function truncate(text, maxLength) {
+	if (text.length <= maxLength) {
+		return text;
+	}
+
+	return `${text.slice(0, maxLength - 1).trimEnd()}...`;
+}
+
+function cleanMarkdownInline(text) {
+	return normalizeWhitespace(
+		text
+			.replace(/!\[[^\]]*]\([^)]+\)/g, "")
+			.replace(/\[([^\]]+)]\([^)]+\)/g, "$1")
+			.replace(/[*_`]/g, "")
+	);
+}
+
+function descriptionFor(content) {
+	const lines = content
+		.split("\n")
+		.map((line) => line.trim())
+		.filter(Boolean)
+		.filter((line) => !line.startsWith("#"))
+		.filter((line) => !line.startsWith("Source:"))
+		.filter((line) => !line.startsWith(":::"))
+		.filter((line) => !line.startsWith("import "));
+
+	const firstUsefulLine = lines.find((line) => !line.startsWith("|")) || "";
+	return truncate(normalizeWhitespace(firstUsefulLine), 180);
+}
+
+if (!fs.existsSync(CORPUS_PATH)) {
+	throw new Error("Run `npm run build` before generating llms files.");
+}
+
+const { docs } = JSON.parse(fs.readFileSync(CORPUS_PATH, "utf8"));
+
+const generatedAt = new Date().toISOString();
+const llmsLines = [
+	"# Open WebUI Docs",
+	"",
+	"Machine-readable entry points to this documentation:",
+	"",
+	"- /llms.txt - curated index of every docs page with short descriptions.",
+	"- /llms-full.txt - every docs page concatenated into a single Markdown file.",
+	"- /sitemap.xml - canonical HTML page discovery.",
+	"- /api/search?q=YOUR_QUERY - deterministic docs search.",
+	"",
+	"For any docs page, append `.md` for Markdown or `.txt` for plain text.",
+	"When citing sources, cite canonical HTML URLs, not `.md`, `.txt`, search API, sitemap, agents.txt, or llms.txt URLs.",
+	"",
+	`Generated: ${generatedAt}`,
+	"",
+	"## Pages",
+	"",
+];
+
+for (const doc of docs) {
+	llmsLines.push(
+		`- [${cleanMarkdownInline(doc.title)}](${doc.url}): ${descriptionFor(doc.content)}`
+	);
+}
+
+fs.writeFileSync(LLMS_PATH, `${llmsLines.join("\n")}\n`);
+
+const fullLines = [
+	"# Open WebUI Docs Full Corpus",
+	"",
+	"Every docs page concatenated into a single Markdown file.",
+	"When citing sources, cite canonical HTML URLs.",
+	"",
+	`Generated: ${generatedAt}`,
+	"",
+];
+
+for (const doc of docs) {
+	fullLines.push("---", "", doc.content.trim(), "");
+}
+
+fs.writeFileSync(LLMS_FULL_PATH, `${fullLines.join("\n")}\n`);
+
+console.log(`Generated llms.txt and llms-full.txt for ${docs.length} docs pages.`);

static/agents.txt

@@ -5,6 +5,8 @@ This site is agent-readable.
 Start at https://docs.openwebui.com/sitemap.xml to discover docs pages.
 Use https://docs.openwebui.com/api/search?q=YOUR_QUERY for docs search.
 For any docs page, append `.md` for Markdown or `.txt` for plain text.
+Use https://docs.openwebui.com/llms.txt for a compact page index.
+Use https://docs.openwebui.com/llms-full.txt for a full Markdown corpus.
 
 When citing sources, cite the canonical HTML URL, not the `.md`, `.txt`, or
 search API URL.