fix(seo): fix robots.txt, add 404 handling, and create llms.txt

- Fix malformed robots.txt: add User-agent/Allow directives, block AI
  training crawlers (CCBot, Google-Extended, Bytespider,
  Applebot-Extended, meta-externalagent) while allowing retrieval bots
- Add proper 404 handling: generate 404.html in prerender script for
  Cloudflare Pages, add top-level catch-all route with NotFound page
- Create /llms.txt with site description and doc structure for LLMs

Author: caio-pizzol

apps/web/public/llms.txt

@@ -0,0 +1,39 @@
+# ooxml.dev
+
+> The OOXML spec, explained by people who actually implemented it.
+
+ooxml.dev is an interactive reference for ECMA-376 (Office Open XML), the standard behind .docx, .xlsx, and .pptx files. It features live previews, real-world implementation gotchas, and AI-powered spec search. Built by the SuperDoc team (https://www.superdoc.dev).
+
+## Main Pages
+
+- [Home](https://ooxml.dev/): Overview and entry point
+- [Docs](https://ooxml.dev/docs/): Interactive OOXML reference documentation
+- [MCP Server](https://ooxml.dev/mcp/): Connect AI assistants to search 18,000+ ECMA-376 specification chunks via MCP
+- [Spec Explorer](https://ooxml.dev/spec/): Search and browse the ECMA-376 specification with semantic search
+
+## Documentation
+
+### Getting Started
+- [Introduction](https://ooxml.dev/docs/): Learn the basics of OOXML and how to use this reference
+
+### WordprocessingML
+- [Paragraphs](https://ooxml.dev/docs/paragraphs/): Text structure and formatting — paragraphs, runs, and text elements (w:p)
+- [Paragraph Borders](https://ooxml.dev/docs/paragraph-borders/): Border properties, between-border groups, nil/none semantics (w:pBdr)
+- [Tables](https://ooxml.dev/docs/tables/): Table structure, properties, and implementation gotchas (w:tbl)
+- [Styles](https://ooxml.dev/docs/styles/): Style definitions, inheritance, and resolution (styles.xml)
+
+### Guides
+- [Creating Documents](https://ooxml.dev/docs/creating-documents/): Step-by-step guide to creating a valid OOXML document from scratch
+- [Common Gotchas](https://ooxml.dev/docs/common-gotchas/): Real-world implementation issues and how to solve them
+
+## MCP Server
+
+The ooxml.dev MCP server provides AI assistants with access to the ECMA-376 specification. Available tools:
+
+- `search_ecma_spec`: Semantic search across 18,000+ specification chunks
+- `get_section`: Retrieve a specific section by ID (e.g., "17.3.2" for paragraph properties)
+- `list_parts`: Browse the specification structure by part (1-4)
+
+## About
+
+ooxml.dev is built and maintained by the SuperDoc team (https://www.superdoc.dev). SuperDoc is a document engine that implements the OOXML specification. The content on ooxml.dev comes from real-world experience building a full OOXML renderer and editor.

apps/web/public/robots.txt

@@ -1 +1,24 @@
+# https://ooxml.dev robots.txt
+
+User-agent: *
+Allow: /
+
+# AI training crawlers - blocked
+User-agent: CCBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: meta-externalagent
+Disallow: /
+
+# For AI-friendly content, see /llms.txt
+
 Sitemap: https://ooxml.dev/sitemap.xml

apps/web/scripts/prerender.ts

@@ -223,6 +223,29 @@ ${urls.join("\n")}
 </urlset>`;
 }
 
+// --- 404 page ---
+
+function render404Page(): string {
+	const title = "Page Not Found | ooxml.dev";
+	const description = "The page you're looking for doesn't exist.";
+	const content = `<main style="max-width:640px;margin:0 auto;padding:4rem 1rem;text-align:center">
+<h1 style="font-size:2rem;font-weight:bold;margin-bottom:1rem">404 — Page Not Found</h1>
+<p style="margin-bottom:2rem;opacity:0.7">The page you're looking for doesn't exist or has been moved.</p>
+<a href="/" style="color:inherit;text-decoration:underline">Go to homepage</a>
+<span style="margin:0 0.5rem;opacity:0.5">·</span>
+<a href="/docs" style="color:inherit;text-decoration:underline">Browse docs</a>
+</main>`;
+
+	let html = template;
+	html = html.replace(/<title>[^<]*<\/title>/, `<title>${escapeHtml(title)}</title>`);
+	html = html.replace(
+		"</head>",
+		`    <meta name="description" content="${escapeHtml(description)}"/>\n    <meta name="robots" content="noindex"/>\n  </head>`,
+	);
+	html = html.replace('<div id="root"></div>', `<div id="root">${content}</div>`);
+	return html;
+}
+
 // --- Main ---
 
 const paths = getAllPaths();
@@ -239,9 +262,14 @@ for (const path of paths) {
 	console.log(`  ✓ ${path}`);
 }
 
+// Generate 404 page (Cloudflare Pages serves this with 404 status)
+const notFoundHtml = render404Page();
+writeFileSync(resolve(DIST, "404.html"), notFoundHtml);
+console.log(`  ✓ /404.html`);
+
 // Generate sitemap
 const sitemap = generateSitemap(paths);
 writeFileSync(resolve(DIST, "sitemap.xml"), sitemap);
 console.log(`  ✓ /sitemap.xml`);
 
-console.log(`\nPre-rendered ${count} pages + sitemap.`);
+console.log(`\nPre-rendered ${count} pages + 404 + sitemap.`);

apps/web/src/main.tsx

@@ -5,6 +5,7 @@ import { DocsLayout } from "./pages/docs/Layout";
 import { DocsPage } from "./pages/docs/Page";
 import { Home } from "./pages/Home";
 import { Mcp } from "./pages/Mcp";
+import { NotFound } from "./pages/NotFound";
 import { SpecExplorer } from "./pages/SpecExplorer";
 import "./index.css";
 
@@ -23,6 +24,7 @@ const router = createBrowserRouter([
 					{ path: "*", element: <DocsPage /> },
 				],
 			},
+			{ path: "*", element: <NotFound /> },
 		],
 	},
 ]);

apps/web/src/pages/NotFound.tsx

@@ -0,0 +1,37 @@
+import { Link } from "react-router-dom";
+import { Navbar } from "../components/Navbar";
+import { Footer } from "../components/Footer";
+import { useDocumentTitle } from "../hooks/useDocumentTitle";
+
+export function NotFound() {
+	useDocumentTitle("Page Not Found | ooxml.dev");
+
+	return (
+		<div className="min-h-screen bg-[var(--color-bg-primary)]">
+			<Navbar maxWidth />
+
+			<main className="mx-auto max-w-xl px-4 py-24 text-center">
+				<h1 className="mb-4 text-3xl font-bold">404 — Page Not Found</h1>
+				<p className="mb-8 text-[var(--color-text-secondary)]">
+					The page you're looking for doesn't exist or has been moved.
+				</p>
+				<div className="flex justify-center gap-4">
+					<Link
+						to="/"
+						className="rounded-lg bg-[var(--color-accent)] px-5 py-2.5 font-medium text-white transition hover:bg-[var(--color-accent-hover)]"
+					>
+						Go to Homepage
+					</Link>
+					<Link
+						to="/docs"
+						className="rounded-lg border border-[var(--color-border)] px-5 py-2.5 font-medium text-[var(--color-text-primary)] transition hover:bg-[var(--color-bg-secondary)]"
+					>
+						Browse Docs
+					</Link>
+				</div>
+			</main>
+
+			<Footer />
+		</div>
+	);
+}