Add web scraping permissions.

Author: njhenry

site/public/robots.txt

@@ -0,0 +1,48 @@
+# As a condition of accessing this website, you agree to abide by the following
+# content signals:
+#
+# (a)  If a Content-Signal = yes, you may collect content for the corresponding use.
+# (b)  If a Content-Signal = no, you may not collect content for the corresponding use.
+# (c)  If no Content-Signal is present for a use, the operator neither grants nor
+#      restricts permission via Content-Signal for that use.
+#
+# Signals:  search   = building a search index / returning hyperlinks and short excerpts
+#                      (not AI-generated summaries)
+#           ai-input = feeding content into AI models (RAG, grounding, generative answers)
+#           ai-train = training or fine-tuning AI models
+#
+# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF RIGHTS
+# UNDER ARTICLE 4 OF EU DIRECTIVE 2019/790 ON COPYRIGHT IN THE DIGITAL SINGLE MARKET.
+
+# --- Default policy: index for search, no AI training -------------------------
+User-agent: *
+Content-Signal: search=yes,ai-train=no
+Allow: /
+# Build artifacts and map data — no value to any crawler, and the PMTiles layers
+# are large. Keep crawlers out of them.
+Disallow: /assets/
+
+# --- AI training / scraper bots: full opt-out ---------------------------------
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: meta-externalagent
+Disallow: /