Add custom robots.txt to disable AI crawling

Author: mbuffa

robots.txt

@@ -0,0 +1,72 @@
+# Block AI bots and crawlers from indexing
+
+# OpenAI
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+# Anthropic
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+# Google AI
+User-agent: Google-Extended
+Disallow: /
+
+# Common Crawl (used by many AI companies)
+User-agent: CCBot
+Disallow: /
+
+# Cohere
+User-agent: cohere-ai
+Disallow: /
+
+# Meta/Facebook
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+# Bytedance/TikTok
+User-agent: Bytespider
+Disallow: /
+
+# Amazon
+User-agent: Amazonbot
+Disallow: /
+
+# Apple
+User-agent: Applebot-Extended
+Disallow: /
+
+# Perplexity
+User-agent: PerplexityBot
+Disallow: /
+
+# Diffbot
+User-agent: Diffbot
+Disallow: /
+
+# Claude (additional variants)
+User-agent: Claude-Web
+Disallow: /
+
+# Omgili
+User-agent: omgili
+Disallow: /
+
+# Omgilibot
+User-agent: omgilibot
+Disallow: /
+
+# Allow other bots (search engines, etc.)
+User-agent: *
+Disallow:
+
+Sitemap: https://mbuffa.github.io/sitemap.xml