fix(build): resolve F-01 JSON docs build failure

Author: ayhammouda

src/mcp_server_python_docs/__main__.py

@@ -129,6 +129,8 @@ def build_index(versions: str, skip_content: bool) -> None:
         ingest_sphinx_json_dir,
         populate_synonyms,
         rebuild_fts_indexes,
+        write_json_build_requirements,
+        write_sphinx_json_sitecustomize,
     )
     from mcp_server_python_docs.storage.db import (
         assert_fts5_available,
@@ -230,10 +232,21 @@ def build_index(versions: str, skip_content: bool) -> None:
                     )
 
                     # Install remaining Doc/requirements.txt deps
-                    doc_reqs = os.path.join(clone_dir, "Doc", "requirements.txt")
-                    if os.path.exists(doc_reqs):
+                    doc_reqs = Path(clone_dir) / "Doc" / "requirements.txt"
+                    if doc_reqs.exists():
+                        json_doc_reqs = doc_reqs.with_name(
+                            "_json-build-requirements.txt"
+                        )
+                        omitted_reqs = write_json_build_requirements(
+                            doc_reqs, json_doc_reqs
+                        )
+                        if omitted_reqs:
+                            logger.info(
+                                "Omitted HTML-only Sphinx extensions for JSON build: %s",
+                                ", ".join(omitted_reqs),
+                            )
                         subprocess.run(
-                            [pip_path, "install", "-r", doc_reqs],
+                            [pip_path, "install", "-r", str(json_doc_reqs)],
                             check=True,
                             capture_output=True,
                             text=True,
@@ -244,6 +257,14 @@ def build_index(versions: str, skip_content: bool) -> None:
                     sphinx_build = os.path.join(scripts_dir, "sphinx-build")
                     doc_dir = os.path.join(clone_dir, "Doc")
                     json_out = os.path.join(doc_dir, "build", "json")
+                    sphinx_compat_dir = Path(clone_dir) / "_sphinx_json_compat"
+                    write_sphinx_json_sitecustomize(sphinx_compat_dir)
+                    sphinx_env = os.environ.copy()
+                    sphinx_env["PYTHONPATH"] = (
+                        str(sphinx_compat_dir)
+                        if not sphinx_env.get("PYTHONPATH")
+                        else f"{sphinx_compat_dir}{os.pathsep}{sphinx_env['PYTHONPATH']}"
+                    )
 
                     logger.info(
                         "Running sphinx-build -b json for Python %s "
@@ -253,12 +274,14 @@ def build_index(versions: str, skip_content: bool) -> None:
                     result = subprocess.run(
                         [
                             sphinx_build, "-b", "json",
+                            "-D", "html_theme=classic",
                             "-j", "auto",
                             doc_dir, json_out,
                         ],
                         capture_output=True,
                         text=True,
                         cwd=doc_dir,
+                        env=sphinx_env,
                     )
                     if result.returncode != 0:
                         logger.error(

src/mcp_server_python_docs/ingestion/sphinx_json.py

@@ -35,6 +35,80 @@
     "contents",
 }
 
+_HTML_ONLY_SPHINX_REQUIREMENTS = frozenset({
+    "python-docs-theme",
+    "sphinx-notfound-page",
+    "sphinxext-opengraph",
+})
+
+_SPHINX_JSON_SITECUSTOMIZE = '''"""Compatibility patch for disposable Sphinx JSON builds."""
+
+from __future__ import annotations
+
+try:
+    from sphinxcontrib.serializinghtml import jsonimpl
+except Exception:
+    jsonimpl = None
+
+if jsonimpl is not None:
+    _original_default = jsonimpl.SphinxJSONEncoder.default
+
+    def _mcp_json_default(self, obj):
+        if obj.__class__.__name__ == "_TranslationProxy":
+            return str(obj)
+        return _original_default(self, obj)
+
+    jsonimpl.SphinxJSONEncoder.default = _mcp_json_default
+'''
+
+
+def _canonical_requirement_name(line: str) -> str | None:
+    stripped = line.strip()
+    if not stripped or stripped.startswith("#") or stripped.startswith("-"):
+        return None
+
+    match = re.match(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)", line)
+    if match is None:
+        return None
+
+    return re.sub(r"[-_.]+", "-", match.group(1)).lower()
+
+
+def write_json_build_requirements(source_path: Path, output_path: Path) -> list[str]:
+    """Write CPython Doc requirements filtered for Sphinx JSON builds.
+
+    CPython's documentation requirements include optional extensions and a
+    theme package that only support HTML output. If installed, Sphinx can load
+    them during the JSON build path and fail before any .fjson files are
+    written.
+    """
+    filtered_lines: list[str] = []
+    omitted: list[str] = []
+
+    for line in source_path.read_text(encoding="utf-8").splitlines(keepends=True):
+        package_name = _canonical_requirement_name(line)
+        if package_name in _HTML_ONLY_SPHINX_REQUIREMENTS:
+            omitted.append(package_name)
+            continue
+        filtered_lines.append(line)
+
+    output_path.write_text("".join(filtered_lines), encoding="utf-8")
+    return omitted
+
+
+def write_sphinx_json_sitecustomize(output_dir: Path) -> Path:
+    """Write a temporary sitecustomize shim for Sphinx JSON builds.
+
+    Sphinx 8.2's JSON encoder does not serialize ``_TranslationProxy`` objects,
+    even though CPython docs can place them in the page context. The Sphinx venv
+    is disposable, so keep this compatibility patch isolated to the JSON build
+    subprocess via PYTHONPATH instead of mutating installed packages.
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+    sitecustomize_path = output_dir / "sitecustomize.py"
+    sitecustomize_path.write_text(_SPHINX_JSON_SITECUSTOMIZE, encoding="utf-8")
+    return sitecustomize_path
+
 
 def parse_fjson(filepath: Path) -> dict:
     """Load and parse a .fjson file.

tests/test_ingestion.py

@@ -21,8 +21,70 @@
     parse_fjson,
     populate_synonyms,
     rebuild_fts_indexes,
+    write_json_build_requirements,
+    write_sphinx_json_sitecustomize,
 )
 
+
+class TestJsonBuildRequirements:
+    def test_omits_html_only_sphinx_extensions(self, tmp_path):
+        source = tmp_path / "requirements.txt"
+        output = tmp_path / "json-requirements.txt"
+        source.write_text(
+            "\n".join([
+                "# CPython docs requirements",
+                "sphinx==8.2.3",
+                "sphinxext-opengraph>=0.9.1",
+                "python-docs-theme>=2025.8",
+                "sphinx-notfound-page==1.0.0",
+                "-c constraints.txt",
+                "blurb",
+            ])
+            + "\n",
+            encoding="utf-8",
+        )
+
+        omitted = write_json_build_requirements(source, output)
+
+        assert omitted == [
+            "sphinxext-opengraph",
+            "python-docs-theme",
+            "sphinx-notfound-page",
+        ]
+        assert output.read_text(encoding="utf-8") == (
+            "# CPython docs requirements\n"
+            "sphinx==8.2.3\n"
+            "-c constraints.txt\n"
+            "blurb\n"
+        )
+
+    def test_matches_requirement_names_case_and_separator_insensitively(self, tmp_path):
+        source = tmp_path / "requirements.txt"
+        output = tmp_path / "json-requirements.txt"
+        source.write_text(
+            "SphinxExt.OpenGraph>=0.9; python_version >= '3.12'\n"
+            "SPHINX_NOTFOUND_PAGE==1.0\n",
+            encoding="utf-8",
+        )
+
+        omitted = write_json_build_requirements(source, output)
+
+        assert omitted == ["sphinxext-opengraph", "sphinx-notfound-page"]
+        assert output.read_text(encoding="utf-8") == ""
+
+
+class TestSphinxJsonSitecustomize:
+    def test_writes_translation_proxy_json_patch(self, tmp_path):
+        output_dir = tmp_path / "compat"
+
+        sitecustomize = write_sphinx_json_sitecustomize(output_dir)
+
+        assert sitecustomize == output_dir / "sitecustomize.py"
+        content = sitecustomize.read_text(encoding="utf-8")
+        assert "_TranslationProxy" in content
+        assert "SphinxJSONEncoder.default" in content
+
+
 # ── fjson parsing tests (INGR-C-04) ──