We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 24db1c6 commit 743d4f3Copy full SHA for 743d4f3
1 file changed
src/app/robots.ts
@@ -1,8 +1,33 @@
1
import { MetadataRoute } from 'next'
2
3
+// AI/LLM crawlers — explicit allow improves GEO discoverability.
4
+const AI_BOTS = [
5
+ 'GPTBot',
6
+ 'ChatGPT-User',
7
+ 'OAI-SearchBot',
8
+ 'ClaudeBot',
9
+ 'Claude-Web',
10
+ 'PerplexityBot',
11
+ 'Perplexity-User',
12
+ 'Google-Extended',
13
+ 'GoogleOther',
14
+ 'CCBot',
15
+ 'anthropic-ai',
16
+ 'cohere-ai',
17
+ 'Bytespider',
18
+ 'Amazonbot',
19
+ 'DuckAssistBot',
20
+ 'YouBot',
21
+ 'Applebot-Extended',
22
+ 'Meta-ExternalAgent',
23
+]
24
+
25
export default function robots(): MetadataRoute.Robots {
26
return {
- rules: { userAgent: '*', allow: '/' },
27
+ rules: [
28
+ { userAgent: '*', allow: '/' },
29
+ ...AI_BOTS.map((bot) => ({ userAgent: bot, allow: '/' })),
30
+ ],
31
sitemap: 'https://commitmentissues.dev/sitemap.xml',
32
}
33
0 commit comments