Only allow known legitimate bots

Author: pierres

root/robots.txt

@@ -1,12 +1,16 @@
-User-agent: *
-sitemap: https://pkgstats.archlinux.de/sitemap.xml
+User-agent: Googlebot
+Allow: /
+
+User-agent: bingbot
+Allow: /
+
+User-agent: DuckDuckBot
 Allow: /
 
-User-agent: SemrushBot
-User-agent: AhrefsBot
-User-agent: dotbot
-User-agent: MJ12bot
-User-agent: GPTBot
-User-agent: OAI-SearchBot
-User-agent: Bytespider
+User-agent: Applebot
+Allow: /
+
+User-agent: *
 Disallow: /
+
+Sitemap: https://pkgstats.archlinux.de/sitemap.xml