feat: Generate sitemap (#215)

* Add sitemap from marketing site

* Add sitemap generator

* Remove ts dependency for generating pages index

* package upgrades

Author: dogmar

.eslintrc.js

@@ -60,7 +60,7 @@ module.exports = {
   },
   overrides: [
     {
-      files: ['.eslintrc.js', 'next.config.js'],
+      files: ['.eslintrc.js', 'next.config.js', 'index-pages.mjs'],
       parserOptions: {
         project: null,
       },

index-pages.mjs

@@ -0,0 +1,78 @@
+import { Dirent } from 'fs'
+import { readdir, writeFile } from 'fs/promises'
+import path from 'path'
+import { fileURLToPath } from 'url'
+
+const __filename = fileURLToPath(import.meta.url)
+
+const __dirname = path.dirname(__filename)
+
+function writeUrl({ location, lastMod, priority = '0.5' }) {
+  return `  <url>
+    <loc>${process.env.NEXT_PUBLIC_ROOT_URL}/${location}</loc>
+    <lastmod>${lastMod}</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>${priority || '0.5'}</priority>
+  </url>`
+}
+
+function wrapSiteMap(content) {
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${content}
+</urlset>
+`
+}
+
+const ignore = [
+  // sitemaps
+  /^sitemap\.xml.*/,
+  // _nextjs templates
+  // .hidden_files
+  // [nextjs_dynamic_pages]
+  // 404 and 500 error pages
+  /^([_.[]|404|500).*$/,
+]
+
+const pageFilter = (file) => {
+  for (const ig of ignore) {
+    if (file.name.match(ig)) {
+      return false
+    }
+  }
+  if (file?.isDirectory()) {
+    return true
+  }
+  return file.name.match(/\.(ts|tsx|js|jsx|md|mdoc)$/)
+}
+
+const rootDir = __dirname
+
+const PAGES_PATH = '/pages'
+
+async function crawlPages(filePath = '/') {
+  const fullPath = path.join(rootDir, PAGES_PATH, filePath)
+  const files = await readdir(fullPath, { withFileTypes: true })
+
+  const filteredFiles = files.filter(pageFilter)
+
+  const pages = []
+  for (const file of filteredFiles) {
+    if (file.isDirectory()) {
+      pages.push(...(await crawlPages(path.join(filePath, file.name))))
+    } else {
+      let pathname = file.name.split('.').slice(0, -1).join('.')
+      pathname = path.join(filePath, pathname.replace(/(^|\/)index$/g, ''))
+      pages.push({ path: pathname })
+    }
+  }
+
+  return pages
+}
+
+const pagesObj = await crawlPages()
+const pages = JSON.stringify(pagesObj, null, '  ')
+
+writeFile(path.join(rootDir, 'src/generated/pages.json'), pages)
+
+export default {}

package.json

@@ -4,8 +4,10 @@
     "node": "18.16.0"
   },
   "scripts": {
-    "dev": "concurrently \"next dev\" \"graphql-codegen --watch --config codegen.ts \" ",
-    "build": "next build",
+    "dev": "run-s generate:pageindex dev:watch",
+    "dev:watch": "concurrently \"next dev\" \"yarn graphql:codegen:watch\"",
+    "build": "run-s generate:pageindex build:next",
+    "build:next": "next build",
     "start": "next start",
     "lint": "run-p lint:format lint:js lint:css",
     "lint:format": "prettier --check --no-error-on-unmatched-pattern ./{src,pages}/**/*.{js,jsx,ts,tsx,graphql,md,mdpart}",
@@ -15,7 +17,9 @@
     "fix:format": "prettier --write --no-error-on-unmatched-pattern ./{src,pages}/**/*.{js,jsx,ts,tsx,graphql,md,mdpart}",
     "fix:js": "eslint --fix {src,pages} --ext ts,tsx,js,jsx",
     "fix:css": "stylelint src/**/*.css --fix",
-    "graphql:codegen": "graphql-codegen --config codegen.yml"
+    "graphql:codegen": "graphql-codegen --config codegen.yml",
+    "graphql:codegen:watch": "graphql-codegen --watch --config codegen.yml",
+    "generate:pageindex": "node index-pages.mjs"
   },
   "dependencies": {
     "@apollo/client": "3.7.15",
@@ -48,10 +52,10 @@
     "js-yaml": "4.1.0",
     "lodash": "4.17.21",
     "memoize-one": "6.0.0",
-    "next": "13.4.4",
+    "next": "13.4.5",
     "next-compose-plugins": "2.2.1",
     "next-transpile-modules": "10.0.0",
-    "posthog-js": "1.66.1",
+    "posthog-js": "1.68.1",
     "raw-loader": "4.0.2",
     "react": "18.2.0",
     "react-dom": "18.2.0",
@@ -72,14 +76,14 @@
     "@graphql-codegen/client-preset": "4.0.0",
     "@pluralsh/eslint-config-typescript": "2.5.41",
     "@pluralsh/stylelint-config": "1.1.3",
-    "@types/node": "20.2.5",
+    "@types/node": "20.3.1",
     "@types/react": "18.2.12",
     "@types/react-dom": "18.2.5",
     "@types/styled-components": "5.1.26",
     "@typescript-eslint/eslint-plugin": "5.59.11",
-    "concurrently": "8.1.0",
+    "concurrently": "8.2.0",
     "eslint": "8.42.0",
-    "eslint-config-next": "13.4.4",
+    "eslint-config-next": "13.4.5",
     "eslint-config-prettier": "8.8.0",
     "eslint-import-resolver-typescript": "3.5.5",
     "eslint-plugin-import": "2.27.5",

pages/sitemap.xml.ts

@@ -0,0 +1,83 @@
+import { until } from '@open-draft/until'
+
+import { getRepos } from '@src/data/getRepos'
+
+import pages from '../src/generated/pages.json'
+
+const S_MAXAGE = 1 * 60 * 60 // 1s * 60s/m * 60m/h = 1 hour
+const STALE_WHILE_REVALIDATE = S_MAXAGE * 2
+
+function urlTag({
+  location,
+  lastMod,
+  priority = '0.5',
+}: {
+  location: string
+  lastMod: string
+  priority?: string | number
+}) {
+  return `  <url>
+    <loc>${process.env.NEXT_PUBLIC_ROOT_URL}${location}</loc>
+    <lastmod>${lastMod}</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>${priority || '0.5'}</priority>
+  </url>`
+}
+
+function wrapSiteMap(content: string) {
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${content}
+</urlset>
+`
+}
+
+export default function SiteMap() {
+  // getServerSideProps will do the heavy lifting
+}
+
+async function generateSiteMap({
+  repos,
+}: {
+  repos: Awaited<ReturnType<typeof getRepos>>
+}) {
+  const lastMod = new Date().toISOString()
+
+  // We generate the XML sitemap with the posts data
+  const sitemap = wrapSiteMap(
+    `${pages
+      ?.map((page) => urlTag({ location: `${page.path}`, lastMod }))
+      .join('\n')}
+  ${repos
+    ?.map((repo) => urlTag({ location: `/applications/${repo.name}`, lastMod }))
+    .join('\n')}`
+  )
+
+  return sitemap
+}
+
+let cachedSiteMap: string
+
+export async function getServerSideProps({ res }) {
+  // We make an API call to gather the URLs for our site
+  const { data: repos, error: reposError } = await until(() => getRepos())
+
+  let sitemap: string = cachedSiteMap
+
+  if (!reposError) {
+    sitemap = await generateSiteMap({ repos })
+  }
+
+  res.setHeader('Content-Type', 'text/xml')
+  res.setHeader(
+    'Cache-Control',
+    `public, s-maxage=${S_MAXAGE}, stale-while-revalidate=${STALE_WHILE_REVALIDATE}`
+  )
+  // we send the XML to the browser
+  res.write(sitemap)
+  res.end()
+
+  return {
+    props: {},
+  }
+}