feat: sitemap.xml and robots.txt (#122)

Author: LadyBluenotes

docs/src/routes/guide/(2)config.mdx

@@ -47,9 +47,12 @@ interface SolidBaseConfig<ThemeConfig> {
   title?: string;
   titleTemplate?: string;
   description?: string;
+  siteUrl?: string;
   logo?: string;
   issueAutolink?: IssueAutoLinkConfig | false;
   llms?: boolean;
+  sitemap?: boolean | SitemapConfig;
+  robots?: boolean | RobotsConfig;
   lang?: string;
   locales?: Record<string, LocaleConfig<ThemeConfig>>;
   themeConfig?: ThemeConfig;
@@ -73,11 +76,14 @@ There are several options for setting site-wide metadata and behavior. These opt
     title: "My Documentation Site",
     titleTemplate: "%s - MySite",
     description: "A comprehensive guide to MySite.",
+    siteUrl: "https://docs.example.com",
     logo: "/logo.png",
 }
 // ..
 ```
 
+Set `siteUrl` to the canonical public URL for your site. SolidBase uses it as the shared base for generated sitemap URLs, `robots.txt`, and default Open Graph URL metadata.
+
 :::note
 For multilingual support, you can use the `locales` option to define different configurations for each locale. More details can be found in the [Internationalization guide](/guide/features/i18n).
 :::
@@ -90,18 +96,65 @@ In addition to setting the site title and description, you can also configure ot
 - `lastUpdated`: An object to configure the display of the last updated timestamp on each page. Set to `false` to disable this feature.
 - `issueAutolink`: An object to configure automatic linking of issue references in your markdown content. Set to `false` to disable this feature.
 - `llms`: Set to `true` to emit an `llms.txt` index and markdown copies of your routes for LLM-friendly documentation output.
+- `sitemap`: Set to `true` to emit `/sitemap.xml`. You can also pass an object to override the hostname or control sitemap chunking for very large sites.
+- `robots`: Set to `true` to emit a simple Google-friendly `/robots.txt` file that allows crawling and points to your sitemap.
 
 ```ts title="app.config.ts"
 // ..
+siteUrl: "https://docs.example.com"
 editPath: "https://github.com/[USERNAME]/[REPO]/edit/main/docs/:path"
 lastUpdated: false;
 issueAutolink: "https://github.com/[USERNAME]/[REPO]/issues/:issue"
 llms: true;
+sitemap: true;
+robots: true;
 // ..
 ```
 
 See the [LLMs.txt guide](/guide/features/llms) for page-level exclusion and output details.
 
+## Sitemap and robots
+
+SolidBase can generate both `/sitemap.xml` and `/robots.txt` during the build.
+
+```ts title="app.config.ts"
+// ..
+siteUrl: "https://docs.example.com",
+sitemap: true,
+robots: true,
+// ..
+```
+
+This emits:
+
+- `/sitemap.xml` with fully-qualified canonical URLs for your markdown routes
+- `hreflang` alternates for localized routes, plus `x-default` when a default-locale route exists
+- `/robots.txt` with an allow-all policy and a `Sitemap:` reference
+
+If you need more control, `sitemap` also accepts an object:
+
+```ts title="app.config.ts"
+// ..
+siteUrl: "https://docs.example.com",
+sitemap: {
+    maxUrlsPerSitemap: 10000,
+},
+robots: {
+    rules: [
+        {
+            userAgent: "*",
+            allow: ["/"],
+            disallow: ["/admin/"],
+        },
+    ],
+},
+// ..
+```
+
+Use `siteUrl` for the normal case. Override `sitemap.hostname` only when your sitemap should use a different canonical host than the rest of the site.
+
+See the [Sitemap and robots guide](/guide/features/sitemap-and-robots) for a full walkthrough of localized alternates, page exclusion, and custom crawler rules.
+
 ### Markdown Configuration
 
 In addition to the default markdown support provided by SolidBase, you can include additional markdown plugins, configurations, and other options through the `markdown` option.

docs/src/routes/guide/features/(4)sitemap-and-robots.mdx

@@ -0,0 +1,123 @@
+---
+title: Sitemap and Robots
+---
+
+# {frontmatter.title}
+
+SolidBase can generate both `/sitemap.xml` and `/robots.txt` during the build so search engines can discover and crawl your docs more reliably.
+
+## Enable the feature
+
+Set a canonical `siteUrl`, then turn both features on in your SolidBase config:
+
+```ts title="app.config.ts"
+import { createSolidBase, defineTheme } from "@kobalte/solidbase/config";
+import defaultTheme from "@kobalte/solidbase/default-theme";
+
+const theme = defineTheme({
+	componentsPath: import.meta.resolve("./src/solidbase-theme"),
+	extends: defaultTheme,
+});
+
+const solidBase = createSolidBase(theme);
+
+export default {
+	...solidBase.startConfig({
+		ssr: true,
+	}),
+	plugins: [
+		solidBase.plugin({
+			title: "My Docs",
+			description: "Documentation for my project",
+			siteUrl: "https://docs.example.com",
+			sitemap: true,
+			robots: true,
+		}),
+	],
+};
+```
+
+With that config, SolidBase emits:
+
+- `/sitemap.xml` with fully-qualified canonical URLs
+- `/robots.txt` with an allow-all policy and a `Sitemap:` reference
+
+## What goes into the sitemap
+
+SolidBase builds the sitemap from markdown routes under `src/routes`.
+
+- `.md` and `.mdx` routes are included automatically
+- `[...404]` routes are excluded
+- localized routes include `hreflang` alternates
+- multilingual route groups include `x-default` when a default-locale page exists
+
+If your site grows large enough, SolidBase automatically switches from a single `sitemap.xml` file to a sitemap index plus chunked sitemap files.
+
+## Localized URLs
+
+When you use `lang` and `locales`, SolidBase groups equivalent localized pages together in the sitemap.
+
+```ts title="app.config.ts"
+// ..
+siteUrl: "https://docs.example.com",
+lang: "en-US",
+locales: {
+	fr: {
+		label: "Français",
+		lang: "fr-FR",
+	},
+},
+sitemap: true,
+// ..
+```
+
+That means a page like `/guide/getting-started` can emit alternates for both `en-US` and `fr-FR`, plus `x-default` pointing at the default-locale URL.
+
+## Excluding pages from the sitemap
+
+You can exclude individual pages with frontmatter:
+
+```md title="src/routes/internal.mdx"
+---
+title: Internal
+sitemap:
+  exclude: true
+---
+```
+
+You can also disable sitemap inclusion entirely for a page with `sitemap: false`.
+
+## Custom sitemap and robots config
+
+The default setup is enough for most sites. If you need more control, you can pass config objects instead of booleans.
+
+```ts title="app.config.ts"
+// ..
+siteUrl: "https://docs.example.com",
+sitemap: {
+	maxUrlsPerSitemap: 10000,
+},
+robots: {
+	rules: [
+		{
+			userAgent: "*",
+			allow: ["/"],
+			disallow: ["/admin/"],
+		},
+	],
+},
+// ..
+```
+
+Use `siteUrl` for the normal case. Only override `sitemap.hostname` when your sitemap should use a different canonical host than the rest of the site.
+
+## Google-friendly defaults
+
+SolidBase keeps the generated files intentionally conservative:
+
+- sitemap URLs are absolute and canonical
+- `priority` and `changefreq` are omitted
+- `lastmod` is omitted unless SolidBase can source it accurately in the future
+- `robots.txt` allows crawling by default instead of blocking resources automatically
+
+If you need page-level metadata details, see the [Frontmatter reference](/reference/frontmatter). For the full config surface, see [Configure Your App](/guide/config).

docs/src/routes/guide/index.mdx

@@ -19,6 +19,10 @@ Out of the box support for Markdown and MDX with almost all extensions you'd eve
 
 Generate an `llms.txt` index and markdown mirrors of your docs for AI tooling and machine-readable documentation workflows. Learn more in the [LLMs.txt guide](/guide/features/llms).
 
+### Search engine discovery
+
+Generate a sitemap and `robots.txt` with localized alternates and sensible defaults for search crawlers. Learn more in the [Sitemap and robots guide](/guide/features/sitemap-and-robots).
+
 ### Customizable
 
 All the SolidBase components can be overridden to transform your website into your own unique design. To see how, visit the [Extending Themes](/guide/customization/extending-themes) documentation.

docs/src/routes/reference/frontmatter.mdx

@@ -49,3 +49,17 @@ llms:
 ```
 
 You can also write `llms: false` to exclude the page entirely.
+
+## Sitemap
+
+The `sitemap` frontmatter key controls whether a page is included in generated sitemap output.
+
+```md
+---
+title: About
+sitemap:
+  exclude: true
+---
+```
+
+You can also write `sitemap: false` to exclude the page entirely.

docs/src/solidbase-theme/Layout.tsx

@@ -70,7 +70,9 @@ function OpenGraph() {
 				name="og:url"
 				content={new URL(
 					location.pathname,
-					import.meta.env.VITE_ORIGIN ?? "https://solidbase.netlify.app",
+					solidBaseCtx.config().siteUrl ??
+						import.meta.env.VITE_ORIGIN ??
+						"https://solidbase.netlify.app",
 				).toString()}
 			/>
 			<Meta name="twitter:card" content="summary_large_image" />

docs/vite.config.mts

@@ -23,7 +23,10 @@ export default defineConfig({
 			title: "SolidBase",
 			description:
 				"Fully featured, fully customisable static site generation for SolidStart",
+			siteUrl: "https://solidbase.dev",
 			llms: true,
+			sitemap: true,
+			robots: true,
 			issueAutolink: "https://github.com/kobaltedev/solidbase/issues/:issue",
 			lang: "en",
 			markdown: {

src/config/index.ts

@@ -13,7 +13,10 @@ export interface SolidBaseConfig<ThemeConfig> {
 	title?: string;
 	titleTemplate?: string;
 	description?: string;
+	siteUrl?: string;
 	llms?: boolean;
+	sitemap?: boolean | SitemapConfig;
+	robots?: boolean | RobotsConfig;
 	logo?: string;
 	issueAutolink?: IssueAutoLinkConfig | false;
 	lang?: string;
@@ -33,6 +36,8 @@ type ResolvedConfigKeys =
 	| "title"
 	| "description"
 	| "llms"
+	| "sitemap"
+	| "robots"
 	| "lang"
 	| "issueAutolink"
 	| "lastUpdated";
@@ -50,6 +55,22 @@ export type LocaleConfig<ThemeConfig> = {
 	themeConfig?: ThemeConfig;
 };
 
+export type SitemapConfig = {
+	hostname?: string;
+	maxUrlsPerSitemap?: number;
+};
+
+export type RobotsRule = {
+	userAgent: string | string[];
+	allow?: string[];
+	disallow?: string[];
+};
+
+export type RobotsConfig = {
+	rules?: RobotsRule[];
+	sitemap?: string | false;
+};
+
 export type ThemeDefinition<Config> = {
 	componentsPath: string;
 	extends?: ThemeDefinition<Config>;
@@ -70,6 +91,8 @@ export function createSolidBase<ThemeConfig>(
 			description:
 				"Fully featured, fully customisable static site generation for SolidStart",
 			llms: false,
+			sitemap: false,
+			robots: false,
 			lang: "en-US",
 			issueAutolink: false,
 			lastUpdated: { dateStyle: "short", timeStyle: "short" },
@@ -118,3 +141,23 @@ export function defineTheme<C>(def: ThemeDefinition<C>) {
 	return def;
 }
 export type Theme<C> = ReturnType<typeof defineTheme<C>>;
+
+export function normalizeSiteUrl(siteUrl: string) {
+	return siteUrl.endsWith("/") ? siteUrl : `${siteUrl}/`;
+}
+
+export function getSiteUrl(config: Pick<SolidBaseConfig<any>, "siteUrl">) {
+	if (!config.siteUrl) return undefined;
+	return normalizeSiteUrl(config.siteUrl);
+}
+
+export function getSitemapHostname(
+	config: Pick<SolidBaseConfig<any>, "siteUrl" | "sitemap">,
+) {
+	if (!config.sitemap) return undefined;
+	if (config.sitemap !== true && config.sitemap.hostname) {
+		return normalizeSiteUrl(config.sitemap.hostname);
+	}
+
+	return getSiteUrl(config);
+}