feat: auto generate markdown version of documents (#3688)

Author: farnabaz

docs/nuxt.config.ts

@@ -53,6 +53,9 @@ export default defineNuxtConfig({
       title: 'Complete Documentation',
       description: 'The complete documentation including all content',
     },
+    contentRawMD: {
+      excludeCollections: ['landing'],
+    },
   },
   studio: {
     route: '/admin',

src/features/llms/module.ts

@@ -1,5 +1,5 @@
 import defu from 'defu'
-import { createResolver, defineNuxtModule, addTypeTemplate, addServerPlugin } from '@nuxt/kit'
+import { createResolver, defineNuxtModule, addTypeTemplate, addServerPlugin, addServerHandler } from '@nuxt/kit'
 
 export default defineNuxtModule({
   meta: {
@@ -10,13 +10,36 @@ export default defineNuxtModule({
     const { resolve } = createResolver(import.meta.url)
 
     addServerPlugin(resolve('runtime/server/content-llms.plugin'))
+    if ((nuxt.options as unknown as { llms: { contentRawMD: false | { excludeCollections: string[] } } })?.llms?.contentRawMD !== false) {
+      addServerHandler({ route: '/raw/**:slug.md', handler: resolve('runtime/server/routes/raw/[...slug].md.get') })
+    }
+
+    nuxt.hook('modules:done', () => {
+      // @ts-expect-error -- TODO: fix types
+      nuxt.options.llms ||= {}
+      // @ts-expect-error -- TODO: fix types
+      nuxt.options.llms.contentRawMD = defu(nuxt.options.llms.contentRawMD, {
+        excludeCollections: [],
+      })
+
+      nuxt.options.runtimeConfig.llms ||= {}
+      // @ts-expect-error -- TODO: fix types
+      nuxt.options.runtimeConfig.llms.contentRawMD = defu(nuxt.options.llms.contentRawMD, {
+        excludeCollections: [],
+      })
+    })
 
     const typeTemplate = addTypeTemplate({
       filename: 'content/llms.d.ts' as `${string}.d.ts`,
       getContents: () => {
         return `
 import type { SQLOperator, PageCollections, PageCollectionItemBase } from '@nuxt/content'
 declare module 'nuxt-llms' {
+  interface ModuleOptions {
+    contentRawMD?: false | {
+      excludeCollections?: string[]
+    }
+  }
   interface LLMsSection {
     contentCollection?: keyof PageCollections
     contentFilters?: Array<{

src/features/llms/runtime/server/content-llms.plugin.ts

@@ -1,12 +1,15 @@
+import { appendHeader } from 'h3'
 import { withBase } from 'ufo'
 import type { NitroApp } from 'nitropack/types'
 import type { ContentLLMSCollectionSection } from './utils'
 import { createDocumentGenerator, prepareContentSections } from './utils'
 import type { PageCollectionItemBase } from '@nuxt/content'
 // @ts-expect-error - typecheck does not detect defineNitroPlugin in imports
 import { defineNitroPlugin, queryCollection } from '#imports'
+import type { ModuleOptions } from 'nuxt-llms'
 
 export default defineNitroPlugin((nitroApp: NitroApp) => {
+  const prerenderPaths = new Set<string>()
   nitroApp.hooks.hook('llms:generate', async (event, options) => {
     prepareContentSections(options.sections)
 
@@ -38,7 +41,7 @@ export default defineNitroPlugin((nitroApp: NitroApp) => {
         return {
           title: doc.title || doc?.seo?.title || '',
           description: doc.description || doc?.seo?.description || '',
-          href: withBase(doc.path, options.domain),
+          href: getDocumentLink(doc.path, section.contentCollection!, options),
         }
       }))
     }
@@ -78,4 +81,29 @@ export default defineNitroPlugin((nitroApp: NitroApp) => {
       }
     }
   })
+
+  if (['nitro-prerender', 'nitro-dev'].includes(import.meta.preset as string)) {
+    nitroApp.hooks.hook('beforeResponse', (event) => {
+      if (event.path === '/') {
+        appendHeader(event, 'x-nitro-prerender', Array.from(prerenderPaths))
+      }
+    })
+  }
+
+  function getDocumentLink(link: string, collection: string, options: ModuleOptions) {
+    const contentRawMD = (options as unknown as { contentRawMD: false | { excludeCollections: string[] } })?.contentRawMD
+    if (contentRawMD === false || contentRawMD?.excludeCollections?.includes(collection)) {
+      return withBase(link, options.domain)
+    }
+
+    link = `/raw${link}.md`
+
+    if (link.endsWith('/.md')) {
+      link = link.slice(0, -3) + 'index.md'
+    }
+
+    prerenderPaths.add(link)
+
+    return withBase(link, options.domain)
+  }
 })

src/features/llms/runtime/server/routes/raw/[...slug].md.get.ts

@@ -0,0 +1,47 @@
+import { withLeadingSlash } from 'ufo'
+import { stringify } from 'minimark/stringify'
+import { queryCollection } from '@nuxt/content/server'
+import type { Collections, PageCollectionItemBase, ResolvedCollection } from '@nuxt/content'
+import { getRouterParams, eventHandler, createError, setHeader } from 'h3'
+import { useRuntimeConfig } from '#imports'
+import collections from '#content/manifest'
+
+export default eventHandler(async (event) => {
+  const config = useRuntimeConfig(event)
+  const llmsConfig = config.llms as { contentRawMD: false | { excludeCollections: string[] } }
+  const slug = getRouterParams(event)['slug.md']
+  if (!slug?.endsWith('.md') || llmsConfig?.contentRawMD === false) {
+    throw createError({ statusCode: 404, statusMessage: 'Page not found', fatal: true })
+  }
+
+  let path = withLeadingSlash(slug.replace('.md', ''))
+  if (path.endsWith('/index')) {
+    path = path.substring(0, path.length - 6)
+  }
+
+  const excludeCollections = llmsConfig?.contentRawMD?.excludeCollections || []
+  const _collections = Object.entries(collections as Record<string, ResolvedCollection>)
+    .filter(([_key, value]) => value.type === 'page' && !excludeCollections.includes(_key))
+    .map(([key]) => key) as string[]
+
+  let page: PageCollectionItemBase | null = null
+  for (const collection of _collections) {
+    page = await queryCollection(event, collection as keyof Collections).path(path).first() as PageCollectionItemBase | null
+    if (page) {
+      break
+    }
+  }
+
+  if (!page) {
+    throw createError({ statusCode: 404, statusMessage: 'Page not found', fatal: true })
+  }
+
+  // Add title and description to the top of the page if missing
+  if (page.body.value[0]?.[0] !== 'h1') {
+    page.body.value.unshift(['blockquote', {}, page.description])
+    page.body.value.unshift(['h1', {}, page.title])
+  }
+
+  setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
+  return stringify({ ...page.body, type: 'minimark' }, { format: 'markdown/html' })
+})