feat(sitemap): enhance sitemap generation and caching

- Implement background sitemap refresh with database session
- Add caching for sitemap XML responses
- Update cache clearing logic for sitemap-related keys
- Adjust cache control headers for improved performance

Author: MarkusNeusinger

api/cache.py

@@ -149,6 +149,9 @@ def clear_spec_cache(spec_id: str) -> int:
     count += clear_cache_by_pattern("specs_list")  # List might have changed
     count += clear_cache_by_pattern("filter:")  # Filters might be affected
     count += clear_cache_by_pattern("stats")  # Stats might have changed
+    count += clear_cache_by_pattern("sitemap")  # Sitemap includes spec URLs
+    count += clear_cache_by_pattern(f"seo:{spec_id}")  # SEO proxy pages for this spec
+    count += clear_cache_by_pattern(f"og:{spec_id}")  # OG images for this spec
     return count
 
 
@@ -172,6 +175,7 @@ def clear_library_cache(library_id: str) -> int:
     count += clear_cache_by_pattern("libraries")  # List might have changed
     count += clear_cache_by_pattern("filter:")  # Filters might be affected
     count += clear_cache_by_pattern("stats")  # Stats might have changed
+    count += clear_cache_by_pattern("sitemap")  # Sitemap includes library URLs
     return count
 
 

api/main.py

@@ -116,18 +116,18 @@ async def add_cache_headers(request: Request, call_next):
 
     path = request.url.path
 
-    # Static data that rarely changes (5 min cache, stale-while-revalidate for 1 hour)
+    # Static data — changes only on deploy (10 min cache, 1h stale-while-revalidate)
     if path in ("/libraries", "/stats"):
-        response.headers["Cache-Control"] = "public, max-age=300, stale-while-revalidate=3600"
-    # Specs list - moderate caching (2 min cache)
+        response.headers["Cache-Control"] = "public, max-age=600, stale-while-revalidate=3600"
+    # Specs list (5 min cache, 1h stale-while-revalidate)
     elif path == "/specs":
-        response.headers["Cache-Control"] = "public, max-age=120, stale-while-revalidate=600"
-    # Filter endpoint - short cache (30 sec) with stale-while-revalidate
+        response.headers["Cache-Control"] = "public, max-age=300, stale-while-revalidate=3600"
+    # Filter endpoint — most dynamic, moderate cache (1 min, 10 min stale)
     elif path == "/plots/filter":
-        response.headers["Cache-Control"] = "public, max-age=30, stale-while-revalidate=300"
-    # Individual spec details
+        response.headers["Cache-Control"] = "public, max-age=60, stale-while-revalidate=600"
+    # Individual spec details (5 min cache, 1h stale-while-revalidate)
     elif path.startswith("/specs/"):
-        response.headers["Cache-Control"] = "public, max-age=120, stale-while-revalidate=600"
+        response.headers["Cache-Control"] = "public, max-age=300, stale-while-revalidate=3600"
 
     return response
 

api/routers/seo.py

@@ -7,9 +7,11 @@
 from fastapi.responses import HTMLResponse, Response
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from api.cache import cache_key, get_cache, set_cache
+from api.cache import cache_key, get_cache, get_or_set_cache, set_cache
 from api.dependencies import optional_db
+from core.config import settings
 from core.database import SpecRepository
+from core.database.connection import get_db_context
 
 
 router = APIRouter(tags=["seo"])
@@ -20,6 +22,42 @@ def _lastmod(dt: datetime | None) -> str:
     return f"<lastmod>{dt.strftime('%Y-%m-%d')}</lastmod>" if dt else ""
 
 
+def _build_sitemap_xml(specs: list) -> str:
+    """Build sitemap XML string from specs."""
+    xml_lines = [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
+        "  <url><loc>https://pyplots.ai/</loc></url>",
+        "  <url><loc>https://pyplots.ai/catalog</loc></url>",
+        "  <url><loc>https://pyplots.ai/mcp</loc></url>",
+        "  <url><loc>https://pyplots.ai/legal</loc></url>",
+    ]
+
+    for spec in specs:
+        if spec.impls:
+            spec_id = html.escape(spec.id)
+            xml_lines.append(f"  <url><loc>https://pyplots.ai/{spec_id}</loc>{_lastmod(spec.updated)}</url>")
+            for impl in spec.impls:
+                library_id = html.escape(impl.library_id)
+                xml_lines.append(
+                    f"  <url><loc>https://pyplots.ai/{spec_id}/{library_id}</loc>{_lastmod(impl.updated)}</url>"
+                )
+
+    xml_lines.append("</urlset>")
+    return "\n".join(xml_lines)
+
+
+_STATIC_SITEMAP = _build_sitemap_xml([])
+
+
+async def _refresh_sitemap() -> str:
+    """Standalone factory for background sitemap refresh (creates own DB session)."""
+    async with get_db_context() as db:
+        repo = SpecRepository(db)
+        specs = await repo.get_all()
+    return _build_sitemap_xml(specs)
+
+
 # Minimal HTML template for social media bots (meta tags are what matters)
 BOT_HTML_TEMPLATE = """<!DOCTYPE html>
 <html lang="en">
@@ -66,41 +104,17 @@ async def get_sitemap(db: AsyncSession | None = Depends(optional_db)):
 
     Includes root, catalog page, and all specs with implementations.
     """
-    key = cache_key("sitemap_xml")
-    cached = get_cache(key)
-    if cached:
-        return Response(content=cached, media_type="application/xml")
-
-    # Build XML lines
-    xml_lines = [
-        '<?xml version="1.0" encoding="UTF-8"?>',
-        '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">',
-        "  <url><loc>https://pyplots.ai/</loc></url>",
-        "  <url><loc>https://pyplots.ai/catalog</loc></url>",
-        "  <url><loc>https://pyplots.ai/mcp</loc></url>",
-        "  <url><loc>https://pyplots.ai/legal</loc></url>",
-    ]
+    if db is None:
+        return Response(content=_STATIC_SITEMAP, media_type="application/xml")
 
-    # Add spec URLs (overview + all implementations)
-    if db is not None:
+    async def _fetch() -> str:
         repo = SpecRepository(db)
         specs = await repo.get_all()
-        for spec in specs:
-            if spec.impls:  # Only include specs with implementations
-                spec_id = html.escape(spec.id)
-                # Overview page
-                xml_lines.append(f"  <url><loc>https://pyplots.ai/{spec_id}</loc>{_lastmod(spec.updated)}</url>")
-                # Individual implementation pages
-                for impl in spec.impls:
-                    library_id = html.escape(impl.library_id)
-                    xml_lines.append(
-                        f"  <url><loc>https://pyplots.ai/{spec_id}/{library_id}</loc>{_lastmod(impl.updated)}</url>"
-                    )
-
-    xml_lines.append("</urlset>")
-    xml = "\n".join(xml_lines)
+        return _build_sitemap_xml(specs)
 
-    set_cache(key, xml)
+    xml = await get_or_set_cache(
+        cache_key("sitemap_xml"), _fetch, refresh_after=settings.cache_refresh_after, refresh_factory=_refresh_sitemap
+    )
     return Response(content=xml, media_type="application/xml")
 
 

tests/unit/api/test_cache.py

@@ -155,6 +155,10 @@ def test_clear_spec_entries(self) -> None:
         set_cache("specs_list", "all specs")
         set_cache("filter:lib=matplotlib", "filter result")
         set_cache("stats", "stats data")
+        set_cache("sitemap_xml", "sitemap")
+        set_cache("seo:scatter-basic", "seo overview")
+        set_cache("seo:scatter-basic:matplotlib", "seo impl")
+        set_cache("og:scatter-basic:matplotlib", "og image")
         set_cache("unrelated:key", "unrelated")
 
         # Clear spec cache
@@ -167,6 +171,10 @@ def test_clear_spec_entries(self) -> None:
         assert get_cache("specs_list") is None
         assert get_cache("filter:lib=matplotlib") is None
         assert get_cache("stats") is None
+        assert get_cache("sitemap_xml") is None
+        assert get_cache("seo:scatter-basic") is None
+        assert get_cache("seo:scatter-basic:matplotlib") is None
+        assert get_cache("og:scatter-basic:matplotlib") is None
 
         # Unrelated should still be there
         assert get_cache("unrelated:key") is not None
@@ -191,6 +199,7 @@ def test_clear_library_entries(self) -> None:
         set_cache("libraries", "all libraries")
         set_cache("filter:lib=matplotlib", "filter result")
         set_cache("stats", "stats data")
+        set_cache("sitemap_xml", "sitemap")
         set_cache("unrelated:key", "unrelated")
 
         # Clear library cache
@@ -202,6 +211,7 @@ def test_clear_library_entries(self) -> None:
         assert get_cache("libraries") is None
         assert get_cache("filter:lib=matplotlib") is None
         assert get_cache("stats") is None
+        assert get_cache("sitemap_xml") is None
 
         # Unrelated should still be there
         assert get_cache("unrelated:key") is not None