Refactor project paths and handling, preserve query params, scope download/XML routes, and adapt template for canonical URLs

html/robots.txt

@@ -1,6 +1,16 @@
-User-agent: SiteimproveBot
-Disallow: /versions
-Disallow: /old_site
-
+# General bots
+# Note: Google does not respect Crawl-delay, but that's fine.
 User-agent: *
-Allow: /
+Allow: /project/*/users/*
+Disallow: /project/
+Disallow: /api/
+Crawl-delay: 10
+
+# AI crawlers — slower, same path rules duplicated
+User-agent: GPTBot
+User-agent: ClaudeBot
+User-agent: PerplexityBot
+Allow: /project/*/users/*
+Disallow: /project/
+Disallow: /api/
+Crawl-delay: 30

models/projects.lua

@@ -117,9 +117,9 @@ local ActiveProjects = Model:extend('active_projects', {
                 '#present:Username=' .. escape(self.username) ..
                 '&ProjectName=' .. escape(self.projectname) ..
                 '&editMode&noRun',
-            download = '/project/' .. escape(self.id),
-            site = '/project?username=' .. escape(self.username) ..
-                '&projectname=' .. escape(self.projectname),
+            download = '/api/v1/project/' .. escape(self.id),
+            site = '/users/' .. escape(self.username) ..
+                '/projects/' .. escape(self.projectname),
             author = '/user?username=' .. escape(self.username),
             embed = 'https://snap.berkeley.edu/embed?projectname=' ..
                 escape(self.projectname) .. '&username=' ..

nginx.conf

@@ -46,6 +46,7 @@ http {
 
     include nginx.conf.d/logging.conf;
     include nginx.conf.d/mime.types;
+    include nginx.conf.d/snap-bot-mitigation.conf;
 
     resolver ${{DNS_RESOLVER}};
 

nginx.conf.d/locations.conf

@@ -1,6 +1,10 @@
 # Shared location configurations for all environments
 # These are included in both the main SSL server block (prod) and non-SSL server block (dev).
 
+# Bot mitigation (robots.txt + UA blocklist). Included here so it is applied
+# to every server block — both prod hosts, both staging hosts, and dev.
+include nginx.conf.d/snap-bot-mitigation-server.conf;
+
 # Specify the cloud domain each page should use.
 
 set_by_lua_block $cloud_url { return os.getenv('CLOUD_URL') }
@@ -48,6 +52,19 @@ location @lapisapp {
     }
 }
 
+# Raw project XML at /project/<numeric_id> — Lapis-served and the largest
+# crawler egress source, so it gets rate-limited via the snap_project zone
+# (defined in snap-bot-mitigation.conf). The upcoming HTML route
+# /project/<name>/users/<user> does not match this regex and is unaffected.
+location ~ ^/project/[0-9]+/?$ {
+    limit_req zone=snap_project burst=120 nodelay;
+    access_log logs/lapis_access.log main_ext if=$should_log;
+    default_type text/html;
+    content_by_lua_block {
+        require("lapis").serve("app")
+    }
+}
+
 # Static content from snapCloud
 # Avoid unnecessarily logging data for CSS, JS, etc.
 location /static/ {

nginx.conf.d/snap-bot-mitigation-server.conf

@@ -0,0 +1,18 @@
+# Bot-mitigation directives that live in server{} context. Included from
+# locations.conf so every server block (prod + staging, both hosts, plus
+# dev) picks them up. The http-context map and limit_req_zone live in
+# nginx.conf.d/snap-bot-mitigation.conf.
+
+# Serve robots.txt as a static file from the repo. Host-independent — does
+# not depend on the per-host static root or the Lapis app.
+location = /robots.txt {
+    access_log off;
+    default_type text/plain;
+    root html;
+}
+
+# 403 any user-agent flagged by $snap_block_ua. `return` is one of the few
+# directives that is safe to use inside `if`.
+if ($snap_block_ua) {
+    return 403;
+}

nginx.conf.d/snap-bot-mitigation.conf

@@ -0,0 +1,24 @@
+# Bot-mitigation directives that must live in the http{} context. The
+# matching server-context directives are in snap-bot-mitigation-server.conf,
+# included from locations.conf.
+
+# UA blocklist for SEO crawlers that ignore robots.txt. \b word-boundaries
+# guard against substring false positives (e.g. "yeti" inside a longer UA).
+map $http_user_agent $snap_block_ua {
+    default          0;
+    "~*ahrefsbot"    1;
+    "~*semrushbot"   1;
+    "~*\bdotbot\b"   1;
+    "~*mj12bot"      1;
+    "~*sleepbot"     1;
+    "~*\byeti\b"     1;
+    "~*blexbot"      1;
+    "~*petalbot"     1;
+}
+
+# Rate-limit zone for the raw XML at /project/<numeric_id>. The rate is
+# deliberately generous because Snap! is used in classrooms where 20–40
+# students share one NAT IP; tighten only with evidence from error.log
+# ("limiting requests" lines).
+limit_req_zone $binary_remote_addr zone=snap_project:10m rate=60r/s;
+limit_req_status 429;