Fix Sitemap (#647)

* Fix Sitemap

* cache sitemap

* sanitize better

Author: evantahler

app/sitemap.ts

@@ -0,0 +1,55 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import type { MetadataRoute } from "next";
+
+const SITE_URL = process.env.SITE_URL ?? "https://docs.arcade.dev";
+const NORMALIZED_SITE_URL = SITE_URL.replace(/\/+$/, "");
+const APP_DIR = path.join(process.cwd(), "app");
+const SKIP_DIRS = new Set(["_meta", "_api", "_redirects", "api"]);
+const INDEX_SUFFIX_REGEX = /\/index$/;
+let cachedRoutes: Promise<MetadataRoute.Sitemap> | null = null;
+
+async function collectRoutes(dir: string): Promise<MetadataRoute.Sitemap> {
+  const dirs = await fs.readdir(dir, { withFileTypes: true });
+  const entries: MetadataRoute.Sitemap = [];
+
+  for (const entry of dirs) {
+    if (entry.isDirectory()) {
+      if (SKIP_DIRS.has(entry.name) || entry.name.includes("[")) {
+        continue;
+      }
+      const childRoutes = await collectRoutes(path.join(dir, entry.name));
+      entries.push(...childRoutes);
+      continue;
+    }
+
+    if (entry.isFile() && entry.name === "page.mdx") {
+      const filePath = path.join(dir, entry.name);
+      const stats = await fs.stat(filePath);
+
+      const relativeDir = path
+        .relative(APP_DIR, dir)
+        .replace(/\\/g, "/")
+        .replace(INDEX_SUFFIX_REGEX, "");
+
+      const routePath = relativeDir ? `/${relativeDir}` : "/";
+      entries.push({
+        url: `${NORMALIZED_SITE_URL}${routePath}`,
+        lastModified: stats.mtime,
+      });
+    }
+  }
+
+  return entries;
+}
+
+export default function sitemap(): Promise<MetadataRoute.Sitemap> {
+  if (!cachedRoutes) {
+    cachedRoutes = collectRoutes(APP_DIR).then((routes) => {
+      routes.sort((a, b) => a.url.localeCompare(b.url));
+      return routes;
+    });
+  }
+
+  return cachedRoutes;
+}

next-env.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
-import "./.next/types/routes.d.ts";
+import "./.next/dev/types/routes.d.ts";
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

next-sitemap.config.js

@@ -10,7 +10,7 @@
     "lint": "pnpm dlx ultracite check",
     "format": "pnpm dlx ultracite fix",
     "prepare": "husky install",
-    "postbuild": "pnpm run custompagefind && next-sitemap",
+    "postbuild": "pnpm run custompagefind",
     "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format",
     "sync:metas": "pnpm dlx tsx scripts/sync-metas.ts app/en",
     "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts",
@@ -40,8 +40,7 @@
     "lucide-react": "0.548.0",
     "mdast-util-to-string": "4.0.0",
     "motion": "12.23.24",
-    "next": "16.0.10",
-    "next-sitemap": "4.2.3",
+    "next": "16.1.1",
     "nextra": "4.6.0",
     "nextra-theme-docs": "4.6.0",
     "posthog-js": "1.298.0",