Merge branch 'staging' into auto/A19S29-merge-master-into-staging

keyman-server · web-flow · commit d3b7f10bdb0a · 2026-05-22T07:26:41.000+02:00
diff --git a/robots.txt b/robots.txt
@@ -1,7 +1,3 @@
-# This bot is too busy (300K+ visits/month across all keyman.com domains)
-User-agent: AhrefsBot
-Disallow: /
-
-# This bot is too busy (300K+ visits/month across all keyman.com domains)
-User-agent: TurnitinBot
+# Disallow all bots since we don't need anything idexed on the staging site
+User-agent: *
 Disallow: /
