feat(seo): update SEO configurations and bot handling

- Add no-cpu-throttling flag in cloudbuild.yaml
- Change robots meta tag to noindex in InteractivePage.tsx
- Update nginx.conf to serve robots.txt directly and adjust bot detection
- Add ignored memory patterns and language server settings in project.yml
- Modify robots.txt to disallow /interactive
- Implement SEO proxy for MCP page in seo.py

Author: MarkusNeusinger

.serena/project.yml

@@ -131,3 +131,17 @@ read_only_memory_patterns: []
 # Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
 # This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
 line_ending:
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
+# No documentation on options means no options are available.
+ls_specific_settings: {}

api/cloudbuild.yaml

@@ -58,6 +58,7 @@ steps:
       - "--execution-environment=gen2"
       - "--set-env-vars=GOOGLE_CLOUD_PROJECT=$PROJECT_ID"
       - "--set-env-vars=GCS_BUCKET=pyplots-images"
+      - "--no-cpu-throttling"
     id: "deploy"
     waitFor: ["push-image"]
 

api/routers/seo.py

@@ -146,6 +146,19 @@ async def seo_legal():
     )
 
 
+@router.get("/seo-proxy/mcp")
+async def seo_mcp():
+    """Bot-optimized MCP page with correct og:tags."""
+    return HTMLResponse(
+        BOT_HTML_TEMPLATE.format(
+            title="MCP Server | pyplots.ai",
+            description="Connect your AI assistant to pyplots via the Model Context Protocol (MCP).",
+            image=DEFAULT_HOME_IMAGE,
+            url="https://pyplots.ai/mcp",
+        )
+    )
+
+
 @router.get("/seo-proxy/{spec_id}")
 async def seo_spec_overview(spec_id: str, db: AsyncSession | None = Depends(optional_db)):
     """Bot-optimized spec overview page with collage og:image."""

app/nginx.conf

@@ -1,5 +1,7 @@
 # Bot detection for SEO - social media crawlers need pre-rendered meta tags
 # These bots cannot execute JavaScript, so we serve them pre-rendered HTML with og:tags
+# NOTE: Search engines (Googlebot, Bingbot, etc.) are NOT included here — they can
+# render JavaScript since 2019 and should see the full SPA for proper indexing.
 map $http_user_agent $is_bot {
     default 0;
     # Social Media
@@ -20,13 +22,6 @@ map $http_user_agent $is_bot {
     ~*skypeuripreview 1;
     ~*microsoft\ teams 1;
     ~*snapchat 1;
-    # Search Engines
-    ~*googlebot 1;
-    ~*bingbot 1;
-    ~*yandexbot 1;
-    ~*duckduckbot 1;
-    ~*baiduspider 1;
-    ~*applebot 1;
     # Link Preview Services
     ~*embedly 1;
     ~*quora\ link\ preview 1;
@@ -71,8 +66,13 @@ server {
         proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
     }
 
+    # Serve robots.txt directly (must not go through bot proxy)
+    location = /robots.txt {
+        try_files $uri =404;
+    }
+
     # SPA routing - serve index.html for all routes
-    # Bots get redirected to backend for proper meta tags
+    # Social media bots get redirected to backend for proper og:tags
     location / {
         # Redirect bots to SEO proxy via error_page trick (nginx-safe pattern)
         error_page 418 = @seo_proxy;

app/public/robots.txt

@@ -1,5 +1,6 @@
 User-agent: *
 Allow: /
 Disallow: /debug
+Disallow: /interactive
 
 Sitemap: https://pyplots.ai/sitemap.xml

app/src/pages/InteractivePage.tsx

@@ -201,7 +201,7 @@ export function InteractivePage() {
         <meta property="og:title" content={`${title} - ${library} (interactive) | pyplots.ai`} />
         <meta property="og:description" content={`Interactive ${title} visualization using ${library}`} />
         <meta property="og:url" content={`https://pyplots.ai/interactive/${specId}/${library}`} />
-        <meta name="robots" content="index, follow" />
+        <meta name="robots" content="noindex, follow" />
         <link rel="canonical" href={`https://pyplots.ai/interactive/${specId}/${library}`} />
       </Helmet>