Added a build step to copy llms.txt to the dist directory for static deployment and cloduflare page function to intercept the requests

Author: treeoflife2

defillama-openapi-free.json

@@ -3096,4 +3096,4 @@
       }
     }
   }
-}
+}

examples/web/functions/_middleware.ts

@@ -0,0 +1,67 @@
+/**
+ * Cloudflare Pages Function to detect AI/LLM requests and serve llms.txt
+ *
+ * This function runs on Cloudflare's edge and intercepts requests before
+ * serving static assets. Place this in /functions/_middleware.ts
+ *
+ * Docs: https://developers.cloudflare.com/pages/functions/middleware/
+ */
+
+const AI_USER_AGENTS = [
+  'anthropic-ai',
+  'claude',
+  'openai',
+  'gptbot',
+  'chatgpt',
+  'bingbot',
+  'googlebot',
+  'google-extended',
+  'perplexitybot',
+  'amazonbot',
+  'meta-externalagent',
+  'cohere-ai',
+  'diffbot',
+  'curl',
+  'wget',
+]
+
+function isAIUserAgent(userAgent: string | null): boolean {
+  if (!userAgent) return false
+  const ua = userAgent.toLowerCase()
+  return AI_USER_AGENTS.some((agent) => ua.includes(agent))
+}
+
+export async function onRequest(context: {
+  request: Request
+  next: () => Promise<Response>
+  env: any
+}): Promise<Response> {
+  const url = new URL(context.request.url)
+  const userAgent = context.request.headers.get('user-agent')
+
+  // Serve llms.txt content on homepage for AI agents
+  if (url.pathname === '/' && isAIUserAgent(userAgent)) {
+    try {
+      // Fetch the llms.txt file from your static assets
+      const llmsTxtUrl = new URL('/llms.txt', context.request.url)
+      const llmsResponse = await fetch(llmsTxtUrl.toString())
+
+      if (llmsResponse.ok) {
+        const llmsContent = await llmsResponse.text()
+        return new Response(llmsContent, {
+          status: 200,
+          headers: {
+            'Content-Type': 'text/plain; charset=utf-8',
+            'X-Served-As': 'llms.txt',
+            'Cache-Control': 'public, max-age=3600',
+          },
+        })
+      }
+    } catch (error) {
+      console.error('Error serving llms.txt:', error)
+    }
+  }
+
+  // Continue to the next middleware or serve static assets
+  return context.next()
+}

examples/web/vite.config.ts

@@ -1,7 +1,7 @@
 import vue from '@vitejs/plugin-vue'
-import { defineConfig, Plugin } from 'vite'
-import path from 'path'
-import fs from 'fs'
+import { defineConfig, type Plugin } from 'vite'
+import { resolve } from 'path'
+import { readFileSync, existsSync, mkdirSync, copyFileSync } from 'fs'
 
 const AI_USER_AGENTS = [
   'anthropic-ai',
@@ -28,12 +28,12 @@ function isAIRequest(userAgent: string | undefined): boolean {
 
 function llmsTxtPlugin(): Plugin {
   let llmsContent: string
+  const llmsPath = resolve(__dirname, '../../llms.txt')
 
   return {
     name: 'llms-txt-plugin',
     configureServer(server) {
-      const llmsPath = path.resolve(__dirname, '../../llms.txt')
-      llmsContent = fs.readFileSync(llmsPath, 'utf-8')
+      llmsContent = readFileSync(llmsPath, 'utf-8')
 
       server.middlewares.use((req, res, next) => {
         const userAgent = req.headers['user-agent']
@@ -54,6 +54,15 @@ function llmsTxtPlugin(): Plugin {
         next()
       })
     },
+    buildEnd() {
+      // Copy llms.txt to dist/public for static deployment
+      const outputDir = resolve(__dirname, 'dist')
+      if (!existsSync(outputDir)) {
+        mkdirSync(outputDir, { recursive: true })
+      }
+      copyFileSync(llmsPath, resolve(outputDir, 'llms.txt'))
+      console.log('✓ Copied llms.txt to dist/')
+    },
   }
 }
 
@@ -64,4 +73,7 @@ export default defineConfig({
     port: 5050,
     open: true,
   },
+  build: {
+    outDir: 'dist',
+  },
 })