feat(sitemap): add routes for robots.txt and sitemap.xml

Author: LadyBluenotes

src/routeTree.gen.ts

@@ -14,7 +14,9 @@ import { Route as TermsRouteImport } from './routes/terms'
 import { Route as TenetsRouteImport } from './routes/tenets'
 import { Route as SupportRouteImport } from './routes/support'
 import { Route as SponsorsEmbedRouteImport } from './routes/sponsors-embed'
+import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml'
 import { Route as RssDotxmlRouteImport } from './routes/rss[.]xml'
+import { Route as RobotsDottxtRouteImport } from './routes/robots[.]txt'
 import { Route as PrivacyRouteImport } from './routes/privacy'
 import { Route as PartnersEmbedRouteImport } from './routes/partners-embed'
 import { Route as PartnersRouteImport } from './routes/partners'
@@ -144,11 +146,21 @@ const SponsorsEmbedRoute = SponsorsEmbedRouteImport.update({
   path: '/sponsors-embed',
   getParentRoute: () => rootRouteImport,
 } as any)
+const SitemapDotxmlRoute = SitemapDotxmlRouteImport.update({
+  id: '/sitemap.xml',
+  path: '/sitemap.xml',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const RssDotxmlRoute = RssDotxmlRouteImport.update({
   id: '/rss.xml',
   path: '/rss.xml',
   getParentRoute: () => rootRouteImport,
 } as any)
+const RobotsDottxtRoute = RobotsDottxtRouteImport.update({
+  id: '/robots.txt',
+  path: '/robots.txt',
+  getParentRoute: () => rootRouteImport,
+} as any)
 const PrivacyRoute = PrivacyRouteImport.update({
   id: '/privacy',
   path: '/privacy',
@@ -710,7 +722,9 @@ export interface FileRoutesByFullPath {
   '/partners': typeof PartnersRoute
   '/partners-embed': typeof PartnersEmbedRoute
   '/privacy': typeof PrivacyRoute
+  '/robots.txt': typeof RobotsDottxtRoute
   '/rss.xml': typeof RssDotxmlRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/sponsors-embed': typeof SponsorsEmbedRoute
   '/support': typeof SupportRoute
   '/tenets': typeof TenetsRoute
@@ -816,7 +830,9 @@ export interface FileRoutesByTo {
   '/partners': typeof PartnersRoute
   '/partners-embed': typeof PartnersEmbedRoute
   '/privacy': typeof PrivacyRoute
+  '/robots.txt': typeof RobotsDottxtRoute
   '/rss.xml': typeof RssDotxmlRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/sponsors-embed': typeof SponsorsEmbedRoute
   '/support': typeof SupportRoute
   '/tenets': typeof TenetsRoute
@@ -925,7 +941,9 @@ export interface FileRoutesById {
   '/partners': typeof PartnersRoute
   '/partners-embed': typeof PartnersEmbedRoute
   '/privacy': typeof PrivacyRoute
+  '/robots.txt': typeof RobotsDottxtRoute
   '/rss.xml': typeof RssDotxmlRoute
+  '/sitemap.xml': typeof SitemapDotxmlRoute
   '/sponsors-embed': typeof SponsorsEmbedRoute
   '/support': typeof SupportRoute
   '/tenets': typeof TenetsRoute
@@ -1038,7 +1056,9 @@ export interface FileRouteTypes {
     | '/partners'
     | '/partners-embed'
     | '/privacy'
+    | '/robots.txt'
     | '/rss.xml'
+    | '/sitemap.xml'
     | '/sponsors-embed'
     | '/support'
     | '/tenets'
@@ -1144,7 +1164,9 @@ export interface FileRouteTypes {
     | '/partners'
     | '/partners-embed'
     | '/privacy'
+    | '/robots.txt'
     | '/rss.xml'
+    | '/sitemap.xml'
     | '/sponsors-embed'
     | '/support'
     | '/tenets'
@@ -1252,7 +1274,9 @@ export interface FileRouteTypes {
     | '/partners'
     | '/partners-embed'
     | '/privacy'
+    | '/robots.txt'
     | '/rss.xml'
+    | '/sitemap.xml'
     | '/sponsors-embed'
     | '/support'
     | '/tenets'
@@ -1364,7 +1388,9 @@ export interface RootRouteChildren {
   PartnersRoute: typeof PartnersRoute
   PartnersEmbedRoute: typeof PartnersEmbedRoute
   PrivacyRoute: typeof PrivacyRoute
+  RobotsDottxtRoute: typeof RobotsDottxtRoute
   RssDotxmlRoute: typeof RssDotxmlRoute
+  SitemapDotxmlRoute: typeof SitemapDotxmlRoute
   SponsorsEmbedRoute: typeof SponsorsEmbedRoute
   SupportRoute: typeof SupportRoute
   TenetsRoute: typeof TenetsRoute
@@ -1448,13 +1474,27 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof SponsorsEmbedRouteImport
       parentRoute: typeof rootRouteImport
     }
+    '/sitemap.xml': {
+      id: '/sitemap.xml'
+      path: '/sitemap.xml'
+      fullPath: '/sitemap.xml'
+      preLoaderRoute: typeof SitemapDotxmlRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/rss.xml': {
       id: '/rss.xml'
       path: '/rss.xml'
       fullPath: '/rss.xml'
       preLoaderRoute: typeof RssDotxmlRouteImport
       parentRoute: typeof rootRouteImport
     }
+    '/robots.txt': {
+      id: '/robots.txt'
+      path: '/robots.txt'
+      fullPath: '/robots.txt'
+      preLoaderRoute: typeof RobotsDottxtRouteImport
+      parentRoute: typeof rootRouteImport
+    }
     '/privacy': {
       id: '/privacy'
       path: '/privacy'
@@ -2379,7 +2419,9 @@ const rootRouteChildren: RootRouteChildren = {
   PartnersRoute: PartnersRoute,
   PartnersEmbedRoute: PartnersEmbedRoute,
   PrivacyRoute: PrivacyRoute,
+  RobotsDottxtRoute: RobotsDottxtRoute,
   RssDotxmlRoute: RssDotxmlRoute,
+  SitemapDotxmlRoute: SitemapDotxmlRoute,
   SponsorsEmbedRoute: SponsorsEmbedRoute,
   SupportRoute: SupportRoute,
   TenetsRoute: TenetsRoute,

src/routes/robots[.]txt.ts

@@ -0,0 +1,25 @@
+import { createFileRoute } from '@tanstack/react-router'
+import { setResponseHeader } from '@tanstack/react-start/server'
+import { generateRobotsTxt, getSiteOrigin } from '~/utils/sitemap'
+
+export const Route = createFileRoute('/robots.txt')({
+  server: {
+    handlers: {
+      GET: async ({ request }: { request: Request }) => {
+        const content = generateRobotsTxt(getSiteOrigin(request))
+
+        setResponseHeader('Content-Type', 'text/plain; charset=utf-8')
+        setResponseHeader(
+          'Cache-Control',
+          'public, max-age=300, must-revalidate',
+        )
+        setResponseHeader(
+          'CDN-Cache-Control',
+          'max-age=3600, stale-while-revalidate=3600',
+        )
+
+        return new Response(content)
+      },
+    },
+  },
+})

src/routes/sitemap[.]xml.ts

@@ -0,0 +1,25 @@
+import { createFileRoute } from '@tanstack/react-router'
+import { setResponseHeader } from '@tanstack/react-start/server'
+import { generateSitemapXml, getSiteOrigin } from '~/utils/sitemap'
+
+export const Route = createFileRoute('/sitemap.xml')({
+  server: {
+    handlers: {
+      GET: async ({ request }: { request: Request }) => {
+        const content = generateSitemapXml(getSiteOrigin(request))
+
+        setResponseHeader('Content-Type', 'application/xml; charset=utf-8')
+        setResponseHeader(
+          'Cache-Control',
+          'public, max-age=300, must-revalidate',
+        )
+        setResponseHeader(
+          'CDN-Cache-Control',
+          'max-age=3600, stale-while-revalidate=3600',
+        )
+
+        return new Response(content)
+      },
+    },
+  },
+})

src/utils/sitemap.ts

@@ -0,0 +1,120 @@
+import { libraries } from '~/libraries'
+import { getPublishedPosts } from '~/utils/blog'
+import { env } from '~/utils/env'
+
+export type SitemapEntry = {
+  path: string
+  lastModified?: string
+}
+
+const HIGH_VALUE_STATIC_SITEMAP_PATHS = [
+  '/',
+  '/blog',
+  '/libraries',
+  '/learn',
+  '/showcase',
+  '/support',
+  '/partners',
+  '/workshops',
+  '/maintainers',
+  '/builder',
+  '/explore',
+  '/ethos',
+  '/tenets',
+] as const satisfies ReadonlyArray<string>
+
+function trimTrailingSlash(url: string) {
+  return url.replace(/\/$/, '')
+}
+
+function escapeXml(value: string) {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+function asLastModified(value: string) {
+  return new Date(`${value}T12:00:00.000Z`).toISOString()
+}
+
+function getLibraryEntries(): Array<SitemapEntry> {
+  return libraries.flatMap((library) => {
+    if (library.visible === false || !library.latestVersion) {
+      return []
+    }
+
+    const basePath = `/${library.id}/latest`
+    const entries: Array<SitemapEntry> = [{ path: basePath }]
+
+    if (library.defaultDocs) {
+      entries.push({
+        path: `${basePath}/docs/${library.defaultDocs}`,
+      })
+    }
+
+    return entries
+  })
+}
+
+function getBlogEntries(): Array<SitemapEntry> {
+  return getPublishedPosts().map((post) => ({
+    path: `/blog/${post.slug}`,
+    lastModified: asLastModified(post.published),
+  }))
+}
+
+export function getSiteOrigin(request: Request) {
+  return trimTrailingSlash(env.SITE_URL || new URL(request.url).origin)
+}
+
+export function getSitemapEntries(): Array<SitemapEntry> {
+  const entries = [
+    ...HIGH_VALUE_STATIC_SITEMAP_PATHS.map((path) => ({ path })),
+    ...getLibraryEntries(),
+    ...getBlogEntries(),
+  ]
+
+  return Array.from(
+    new Map(entries.map((entry) => [entry.path, entry])).values(),
+  )
+}
+
+export function generateSitemapXml(origin: string) {
+  const urls = getSitemapEntries()
+    .map((entry) => {
+      const loc = `${origin}${entry.path}`
+
+      return [
+        '  <url>',
+        `    <loc>${escapeXml(loc)}</loc>`,
+        entry.lastModified
+          ? `    <lastmod>${entry.lastModified}</lastmod>`
+          : '',
+        '  </url>',
+      ]
+        .filter(Boolean)
+        .join('\n')
+    })
+    .join('\n')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}
+</urlset>`
+}
+
+export function generateRobotsTxt(origin: string) {
+  return [
+    'User-agent: *',
+    'Allow: /',
+    'Disallow: /admin',
+    'Disallow: /account',
+    'Disallow: /api',
+    'Disallow: /oauth',
+    '',
+    `Sitemap: ${origin}/sitemap.xml`,
+  ].join('\n')
+}