POC for serving the markdown pages (useful for LLMs)

Author: fulopkovacs

app/api.ts

@@ -0,0 +1,7 @@
+// app/api.ts
+import {
+  createStartAPIHandler,
+  defaultAPIFileRouteHandler,
+} from '@tanstack/start/api'
+
+export default createStartAPIHandler(defaultAPIFileRouteHandler)

app/routes/$libraryId/$version.docs.framework.$framework.$.tsx

@@ -1,5 +1,5 @@
 import { seo } from '~/utils/seo'
-import { createFileRoute } from '@tanstack/react-router'
+import { createFileRoute, redirect } from '@tanstack/react-router'
 import { Doc } from '~/components/Doc'
 import { loadDocs } from '~/utils/docs'
 import { getBranch, getLibrary } from '~/libraries'
@@ -15,6 +15,20 @@ export const Route = createFileRoute(
 
     const library = getLibrary(libraryId)
 
+    const isMarkdown = !!docsPath?.endsWith('.md')
+
+    if (isMarkdown) {
+      const href =
+        '/api/md' +
+        ctx.location.pathname.slice(
+          0,
+          ctx.location.pathname.length - '.md'.length
+        )
+      throw redirect({
+        href,
+      })
+    }
+
     return loadDocs({
       repo: library.repo,
       branch: getBranch(library, version),

app/routes/api/md/$libraryId/$version.docs.framework.$framework.$.ts

@@ -0,0 +1,38 @@
+import {  createAPIRoute } from '@tanstack/start/api'
+import { getBranch, getLibrary } from '~/libraries'
+import { loadDocs } from '~/utils/docs'
+
+export const APIRoute = createAPIRoute('/api/md/$libraryId/$version/docs/framework/$framework/$')({
+  GET: async ({ params, request }) => {
+    const { libraryId, version, framework, _splat: docsPath } = params
+    const library = getLibrary(libraryId)
+
+    const location = new URL(request.url)
+
+    const loadDocsArgs = {
+      repo: library.repo,
+      branch: getBranch(library, version),
+      docsPath: `${
+        library.docsRoot || 'docs'
+      }/framework/${framework}/${docsPath}`,
+      currentPath: location.pathname.slice('/api/md'.length),
+      redirectPath: `/${library.id}/${version}/docs/overview`,
+      useServerFn: false
+    }
+
+    const res = await loadDocs(loadDocsArgs)
+
+
+    const { content, description, title} = res
+
+    // Generate or fetch the Markdown content dynamically
+    const markdownContent = `# ${title}\n\n> ${description}\n\n${content}`
+
+    // Return the Markdown content as a response
+    return new Response(markdownContent, {
+      headers: {
+        'Content-Type': 'text/markdown',
+      },
+    })
+  },
+})

app/utils/docs.ts

@@ -9,18 +9,27 @@ import { createServerFn } from '@tanstack/start'
 import { z } from 'zod'
 import { setHeader } from 'vinxi/http'
 
+const FetchDocsDataSchema = z.object({
+  repo: z.string(),
+  branch: z.string(),
+  filePath: z.string(),
+})
+type FetchDocsData = z.infer<typeof FetchDocsDataSchema>
+
 export const loadDocs = async ({
   repo,
   branch,
   // currentPath,
   // redirectPath,
   docsPath,
+  useServerFn = true,
 }: {
   repo: string
   branch: string
   docsPath: string
   currentPath: string
   redirectPath: string
+  useServerFn?: boolean
 }) => {
   if (!branch) {
     throw new Error('Invalid branch')
@@ -32,55 +41,59 @@ export const loadDocs = async ({
 
   const filePath = `${docsPath}.md`
 
-  return await fetchDocs({
+  const params: { data: FetchDocsData } = {
     data: {
       repo,
       branch,
       filePath,
-      // currentPath,
-      // redirectPath,
     },
-  })
+  }
+
+  return useServerFn ? fetchDocs(params) : fetchDocsFunction(params)
 }
 
-export const fetchDocs = createServerFn({ method: 'GET' })
-  .validator(
-    z.object({ repo: z.string(), branch: z.string(), filePath: z.string() })
-  )
-  .handler(async ({ data: { repo, branch, filePath } }) => {
-    const file = await fetchRepoFile(repo, branch, filePath)
+const fetchDocsFunction = async ({
+  data: { repo, branch, filePath },
+}: {
+  data: FetchDocsData
+}) => {
+  const file = await fetchRepoFile(repo, branch, filePath)
+
+  if (!file) {
+    throw notFound()
+    // if (currentPath === redirectPath) {
+    //   // console.log('not found')
+    //   throw notFound()
+    // } else {
+    //   // console.log('redirect')
+    //   throw redirect({
+    //     to: redirectPath,
+    //   })
+    // }
+  }
 
-    if (!file) {
-      throw notFound()
-      // if (currentPath === redirectPath) {
-      //   // console.log('not found')
-      //   throw notFound()
-      // } else {
-      //   // console.log('redirect')
-      //   throw redirect({
-      //     to: redirectPath,
-      //   })
-      // }
-    }
+  const frontMatter = extractFrontMatter(file)
+  const description = removeMarkdown(frontMatter.excerpt ?? '')
 
-    const frontMatter = extractFrontMatter(file)
-    const description = removeMarkdown(frontMatter.excerpt ?? '')
+  // Cache for 5 minutes on shared cache
+  // Revalidate in the background
+  setHeader('Cache-Control', 'public, max-age=0, must-revalidate')
+  setHeader(
+    'CDN-Cache-Control',
+    'max-age=300, stale-while-revalidate=300, durable'
+  )
 
-    // Cache for 5 minutes on shared cache
-    // Revalidate in the background
-    setHeader('Cache-Control', 'public, max-age=0, must-revalidate')
-    setHeader(
-      'CDN-Cache-Control',
-      'max-age=300, stale-while-revalidate=300, durable'
-    )
+  return {
+    title: frontMatter.data?.title,
+    description,
+    filePath,
+    content: frontMatter.content,
+  }
+}
 
-    return {
-      title: frontMatter.data?.title,
-      description,
-      filePath,
-      content: frontMatter.content,
-    }
-  })
+export const fetchDocs = createServerFn({ method: 'GET' })
+  .validator(FetchDocsDataSchema)
+  .handler(fetchDocsFunction)
 
 export const fetchFile = createServerFn({ method: 'GET' })
   .validator(