Bot Serving Check #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Synthetic monitor for the nginx bot -> seo-proxy path. | |
| # | |
| # Humans get the SPA shell and never notice when the bot hop breaks: the site | |
| # looks healthy, Plausible shows normal traffic, CI is green — and every | |
| # crawler sees an error page. Exactly that happened between 2026-06-12 and | |
| # 2026-07-09: the @seo_proxy upstream TLS verification failed (default | |
| # proxy_ssl_verify_depth 1 vs a 4-deep Let's Encrypt chain) and every bot UA | |
| # received HTTP 502 on every page, undetected for ~4 weeks. This check is the | |
| # alarm that was missing. | |
| # | |
| # The checks target the Cloud Run ORIGIN, not https://anyplot.ai: Cloudflare's | |
| # bot management 403s GitHub-runner (datacenter) IPs — including UA-spoofed | |
| # "Googlebot", which only passes verified-bot checks from real Google IPs — | |
| # verified on the first dispatched run. The origin is also exactly the layer | |
| # that broke in the incident above; Cloudflare-edge issues are out of this | |
| # monitor's reach by design. | |
| name: Bot Serving Check | |
| on: | |
| schedule: | |
| - cron: "23 6 * * *" # daily 06:23 UTC | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| bot-serving: | |
| runs-on: ubuntu-latest | |
| # 7 checks x (--retry 2 -> up to 3 attempts x --max-time 30) can reach | |
| # ~10.5 min worst-case; 14 leaves room to report a clean failure. | |
| timeout-minutes: 14 | |
| steps: | |
| - name: Crawler UAs must get 200 + per-route titles | |
| run: | | |
| set -uo pipefail | |
| # Cloud Run origin of the anyplot-app service (see header comment | |
| # for why not https://anyplot.ai). | |
| ORIGIN="https://anyplot-app-r3tvmejsmq-ez.a.run.app" | |
| GOOGLEBOT="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" | |
| TWITTERBOT="Twitterbot/1.0" | |
| CLAUDEBOT="Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" | |
| CHATGPTUSER="Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)" | |
| HUMAN="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" | |
| fail=0 | |
| check() { | |
| local ua="$1" url="$2" expect="$3" | |
| local code | |
| # On curl failure REPLACE the code — a failing curl can still have | |
| # printed a partial -w code; appending would yield e.g. "200000". | |
| code=$(curl -sS --retry 2 --max-time 30 -A "$ua" -o body.html -w '%{http_code}' "$url") || code="000" | |
| if [ "$code" != "200" ]; then | |
| echo "::error::$url with UA '$ua' returned HTTP $code (expected 200)" | |
| fail=1 | |
| elif ! grep -qF "$expect" body.html; then | |
| echo "::error::$url with UA '$ua' returned 200 but the body is missing: $expect" | |
| fail=1 | |
| else | |
| echo "OK: $url ($ua)" | |
| fi | |
| } | |
| # Bot path: prerendered per-route HTML from the seo-proxy | |
| check "$GOOGLEBOT" "$ORIGIN/" "<title>anyplot.ai</title>" | |
| check "$GOOGLEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| check "$TWITTERBOT" "$ORIGIN/scatter-basic/python/matplotlib" "<title>Basic Scatter Plot - Matplotlib | anyplot.ai</title>" | |
| # AI assistants take the same prerendered path (nginx $is_bot). These | |
| # checks hit the ORIGIN, so they verify the nginx map independently of | |
| # whether Cloudflare's AI Crawl Control currently 403s these UAs at | |
| # the edge — an edge-level policy change needs no change here. | |
| check "$CLAUDEBOT" "$ORIGIN/scatter-basic" "<title>Basic Scatter Plot | anyplot.ai</title>" | |
| # llms.txt must be served directly, never proxied to the seo backend — | |
| # including for a mapped crawler UA, which is the whole point of the | |
| # `location = /llms.txt` bypass. | |
| check "$GOOGLEBOT" "$ORIGIN/llms.txt" "# anyplot" | |
| check "$CHATGPTUSER" "$ORIGIN/llms.txt" "# anyplot" | |
| # Control: humans must still get the SPA shell | |
| check "$HUMAN" "$ORIGIN/" '<div id="root">' | |
| exit $fail |