feat: block /debug route from search engine indexing (#3815)

Adds Disallow: /debug to robots.txt to prevent search engines
from crawling and indexing the internal debug dashboard.

This follows SEO best practices for internal admin/debug tools.

---------

Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

api/routers/seo.py

@@ -42,6 +42,17 @@
 DEFAULT_DESCRIPTION = "library-agnostic, ai-powered python plotting."
 
 
+@router.get("/robots.txt")
+async def get_robots():
+    """
+    Serve robots.txt for API backend.
+
+    Blocks all crawlers - APIs should not be indexed by search engines.
+    Social media bots (WhatsApp, Twitter, etc.) are unaffected.
+    """
+    return Response(content="User-agent: *\nDisallow: /\n", media_type="text/plain")
+
+
 @router.get("/sitemap.xml")
 async def get_sitemap(db: AsyncSession | None = Depends(optional_db)):
     """

app/public/robots.txt

@@ -1,4 +1,5 @@
 User-agent: *
 Allow: /
+Disallow: /debug
 
 Sitemap: https://pyplots.ai/sitemap.xml

docs/reference/seo.md

@@ -210,6 +210,34 @@ Uses **MonoLisa** variable font (commercial, not in repo):
 - Cached locally in `/tmp/pyplots-fonts/`
 - Fallback: DejaVuSansMono-Bold
 
+## Robots.txt
+
+### Frontend (pyplots.ai)
+
+Static file at `app/public/robots.txt`:
+
+```txt
+User-agent: *
+Allow: /
+Disallow: /debug
+
+Sitemap: https://pyplots.ai/sitemap.xml
+```
+
+### Backend (api.pyplots.ai)
+
+Dynamic endpoint at `GET /robots.txt`:
+
+```txt
+User-agent: *
+Disallow: /
+```
+
+**Why block the API?**
+- APIs should not be indexed by search engines
+- Prevents crawling of debug endpoints, docs, and API responses
+- Social media bots (WhatsApp, Twitter, etc.) are unaffected - they fetch og:images directly
+
 ## Sitemap
 
 Dynamic XML sitemap for search engine indexing.
@@ -277,7 +305,8 @@ curl -o test.png https://api.pyplots.ai/og/scatter-basic.png
 | File | Purpose |
 |------|---------|
 | `app/nginx.conf` | Bot detection, SPA routing, sitemap proxy |
-| `api/routers/seo.py` | SEO proxy endpoints, sitemap generation |
+| `app/public/robots.txt` | Frontend robots.txt (blocks /debug) |
+| `api/routers/seo.py` | SEO proxy endpoints, robots.txt, sitemap generation |
 | `api/routers/og_images.py` | Branded og:image endpoints |
 | `core/images.py` | Image processing, branding functions |
 

tests/e2e/test_api_postgres.py

@@ -196,6 +196,14 @@ async def test_download_impl_not_found(self, client):
 class TestSeoEndpoints:
     """E2E tests for SEO endpoints with real PostgreSQL."""
 
+    async def test_robots_txt(self, client):
+        """Should return robots.txt blocking crawlers from all routes."""
+        response = await client.get("/robots.txt")
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "text/plain; charset=utf-8"
+        assert "User-agent: *" in response.text
+        assert "Disallow: /" in response.text
+
     async def test_sitemap(self, client):
         """Should return sitemap XML with spec URLs."""
         response = await client.get("/sitemap.xml")

tests/integration/api/test_api_endpoints.py

@@ -193,6 +193,14 @@ async def test_download_impl_not_found(self, client):
 class TestSeoEndpoints:
     """Integration tests for SEO endpoints."""
 
+    async def test_robots_txt(self, client):
+        """Should return robots.txt blocking crawlers from all routes."""
+        response = await client.get("/robots.txt")
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "text/plain; charset=utf-8"
+        assert "User-agent: *" in response.text
+        assert "Disallow: /" in response.text
+
     async def test_sitemap(self, client):
         """Should return sitemap XML with spec URLs."""
         response = await client.get("/sitemap.xml")

tests/unit/api/test_routers.py

@@ -409,6 +409,15 @@ def test_download_gcs_error(self, client: TestClient, mock_spec) -> None:
 class TestSeoRouter:
     """Tests for SEO router."""
 
+    def test_robots_txt(self, client: TestClient) -> None:
+        """robots.txt should block crawlers from all routes."""
+        response = client.get("/robots.txt")
+        assert response.status_code == 200
+        assert response.headers["content-type"] == "text/plain; charset=utf-8"
+        content = response.text
+        assert "User-agent: *" in content
+        assert "Disallow: /" in content
+
     def test_sitemap_structure(self, client: TestClient) -> None:
         """Sitemap should return valid XML structure."""
         with patch(DB_CONFIG_PATCH, return_value=False):