feat: add Markdown MCP docs generator (Docusaurus- and pdoc-compatible) (#1015)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Author: aaronsteers

.gitignore

@@ -11,6 +11,9 @@ viztracer_report.json
 # Packaged docs
 docs/*.zip
 
+# Generated MCP server docs (regenerate via `poe mcp-docs-md`)
+docs/mcp-generated/
+
 # Misc
 .DS_Store
 

airbyte/mcp/cloud.py

@@ -1,5 +1,14 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-"""Airbyte Cloud MCP operations."""
+"""Airbyte Cloud MCP operations.
+
+.. include:: ../../docs/mcp-generated/cloud.md
+"""
+
+# No public Python API — MCP primitives are registered via decorators and
+# documented via the generated Markdown include above. Setting `__all__` to an
+# empty list tells pdoc (and other doc tools) not to surface the individual
+# tool / helper definitions as a redundant "API Documentation" list.
+__all__: list[str] = []
 
 from pathlib import Path
 from typing import Annotated, Any, Literal, cast

airbyte/mcp/local.py

@@ -1,5 +1,14 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-"""Local MCP operations."""
+"""Local MCP operations.
+
+.. include:: ../../docs/mcp-generated/local.md
+"""
+
+# No public Python API — MCP primitives are registered via decorators and
+# documented via the generated Markdown include above. Setting `__all__` to an
+# empty list tells pdoc (and other doc tools) not to surface the individual
+# tool / helper definitions as a redundant "API Documentation" list.
+__all__: list[str] = []
 
 import sys
 import traceback

airbyte/mcp/prompts.py

@@ -3,6 +3,8 @@
 
 This module defines prompts that can be invoked by MCP clients to perform
 common workflows.
+
+.. include:: ../../docs/mcp-generated/prompts.md
 """
 
 from __future__ import annotations
@@ -13,6 +15,13 @@
 from pydantic import Field
 
 
+# No public Python API — MCP primitives are registered via decorators and
+# documented via the generated Markdown include above. Setting `__all__` to an
+# empty list tells pdoc (and other doc tools) not to surface the individual
+# tool / helper definitions as a redundant "API Documentation" list.
+__all__: list[str] = []
+
+
 if TYPE_CHECKING:
     from fastmcp import FastMCP
 

airbyte/mcp/registry.py

@@ -1,5 +1,14 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-"""Airbyte Cloud MCP operations."""
+"""Airbyte connector registry MCP operations.
+
+.. include:: ../../docs/mcp-generated/registry.md
+"""
+
+# No public Python API — MCP primitives are registered via decorators and
+# documented via the generated Markdown include above. Setting `__all__` to an
+# empty list tells pdoc (and other doc tools) not to surface the individual
+# tool / helper definitions as a redundant "API Documentation" list.
+__all__: list[str] = []
 
 # Note: Deferred type evaluation must be avoided due to FastMCP/Pydantic needing
 # types to be available at import time for tool registration.

docs/CONTRIBUTING.md

@@ -143,3 +143,36 @@ poe mcp-serve-sse      # Server-Sent Events transport on localhost:8000
 
 poe mcp-inspect        # Show all available MCP tools and their schemas
 ```
+
+### Generating Markdown docs for the MCP Server
+
+The repo ships a small script (`scripts/generate_mcp_markdown.py`) that
+introspects the MCP server via `fastmcp inspect` and emits a Markdown
+documentation site under `docs/mcp-generated/` (git-ignored). The output is
+plain CommonMark with no MDX-only components, so it is both Docusaurus-hostable
+and consumable by `pdoc` — the four `airbyte.mcp.{cloud,local,registry,prompts}`
+modules pull their respective generated file in via pdoc's `.. include::`
+directive, so `poe docs-generate` surfaces the generated tool docs on each
+module's pdoc page alongside the regular `docs/generated/` output.
+
+```bash
+uv sync --group dev
+poe mcp-docs-md
+```
+
+One Markdown file is produced per MCP module, plus an `index.md`. For the
+PyAirbyte server that is:
+
+- `index.md` — server overview (name, version, instructions, totals, module table)
+- `cloud.md` — tools registered by `airbyte.mcp.cloud`
+- `local.md` — tools registered by `airbyte.mcp.local`
+- `registry.md` — tools registered by `airbyte.mcp.registry`
+- `prompts.md` — prompts registered by `airbyte.mcp.prompts`
+- `misc.md` — anything without an `mcp_module` annotation (currently just the
+  `server_info` resource)
+
+Inside each module page, primitives are grouped by kind (`## Tools`,
+`## Prompts`, `## Resources`), and each primitive has an HTML anchor
+(`<a id="name"></a>`) above its H3 so links like
+`cloud.md#deploy_source_to_cloud` resolve in both pdoc and Docusaurus.
+Regenerate after any change to MCP tool signatures, descriptions, or schemas.

docs/generate.py

@@ -10,20 +10,81 @@
 
 from __future__ import annotations
 
+import importlib.util
 import pathlib
 import shutil
+import sys
 
 import pdoc
+import pdoc.render_helpers
+
+
+def _regenerate_mcp_markdown() -> None:
+    """Regenerate `docs/mcp-generated/` before pdoc runs.
+
+    The `airbyte.mcp.{cloud,local,registry,prompts}` modules pull the
+    per-module Markdown files from `docs/mcp-generated/` via pdoc's
+    `.. include::` directive. That directory is git-ignored, so on a clean
+    checkout pdoc would fail to resolve the include unless we regenerate it
+    here. Running the generator from inside `docs-generate` makes the full
+    docs build reproducible from a fresh clone (and matches the standalone
+    `poe mcp-docs-md` task).
+
+    If generation fails (e.g. `fastmcp` is not installed, or the MCP server
+    import fails), we print a warning and continue: pdoc will still build,
+    and the include directive will just surface the missing file.
+
+    We load the generator via `importlib.util` from its on-disk path rather
+    than a plain `from generate_mcp_markdown import ...`: the generator
+    lives under `scripts/` (not on `sys.path`), and a static import would
+    also trip `deptry` into flagging `generate_mcp_markdown` as a missing
+    external dependency.
+    """
+    script = pathlib.Path(__file__).parent.parent / "scripts" / "generate_mcp_markdown.py"
+    if not script.exists():
+        print(f"[docs-generate] MCP markdown generator not found at {script}; skipping.")
+        return
+    try:
+        spec = importlib.util.spec_from_file_location("_mcp_markdown_gen", script)
+        if spec is None or spec.loader is None:
+            msg = f"Could not load spec for {script}"
+            raise RuntimeError(msg)  # noqa: TRY301
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        print("[docs-generate] Regenerating docs/mcp-generated/ ...")
+        module.generate(
+            server_spec=module.DEFAULT_SERVER_SPEC,
+            output=module.DEFAULT_OUTPUT,
+        )
+    except Exception as ex:
+        print(
+            f"[docs-generate] WARNING: failed to regenerate MCP Markdown docs: {ex}. "
+            "pdoc will continue, but module pages may show missing include warnings.",
+            file=sys.stderr,
+        )
 
 
 def run() -> None:
     """Generate docs for all public modules in PyAirbyte and save them to docs/generated."""
     public_modules = ["airbyte", "airbyte/cli/pyab.py"]
 
+    # Regenerate MCP Markdown first so the `.. include::` directives in the
+    # MCP module docstrings resolve on a clean checkout (docs/mcp-generated/
+    # is git-ignored).
+    _regenerate_mcp_markdown()
+
     # recursively delete the docs/generated folder if it exists
     if pathlib.Path("docs/generated").exists():
         shutil.rmtree("docs/generated")
 
+    # pdoc's default sidebar TOC depth is 2 (H1 + H2 only), which hides the
+    # per-tool H3 anchors produced by our MCP Markdown generator. Bump to 3 so
+    # individual tools / prompts / resources show up in the left nav. This
+    # monkey-patches the module-level `markdown_extensions` dict because pdoc
+    # 16's `configure()` does not expose markdown extension options.
+    # pyrefly: ignore[unsupported-operation]
+    pdoc.render_helpers.markdown_extensions["toc"] = {"depth": 3}
+
     pdoc.render.configure(
         template_directory=pathlib.Path("docs/templates"),
         show_source=True,

docs/templates/custom.css

@@ -174,6 +174,22 @@ nav a:hover {
   color: var(--link-hover) !important;
 }
 
+/*
+ * Progressively indent nested TOC levels in the sidebar.
+ *
+ * pdoc's default layout.css indents *all* non-top-level nav items by a single
+ * (pad + indent) step, which makes H2 and H3 entries render at the same visual
+ * depth. When the generated MCP Markdown uses H3 headings per-tool nested
+ * under an H2 "Tools" heading, we want the tool names to appear visibly
+ * nested under the section heading in the left nav.
+ */
+nav.pdoc > div > ul > li > ul > li > ul > li > a {
+  padding-left: calc(var(--pad) + (var(--indent) * 2)) !important;
+}
+nav.pdoc > div > ul > li > ul > li > ul > li > ul > li > a {
+  padding-left: calc(var(--pad) + (var(--indent) * 3)) !important;
+}
+
 /* Style badges and labels */
 .badge {
   background-color: var(--color-green-40);

pyproject.toml

@@ -174,6 +174,7 @@ mcp-serve-http = { cmd = "python -c \"from airbyte.mcp.server import app; app.ru
 mcp-serve-sse = { cmd = "python -c \"from airbyte.mcp.server import app; app.run(transport='sse', host='127.0.0.1', port=8000)\"", help = "Start the MCP server with SSE transport" }
 mcp-inspect = { cmd = "fastmcp inspect airbyte/mcp/server.py:app", help = "Inspect MCP tools and resources (supports --tools, --health, etc.)" }
 mcp-tool-test = { cmd = "python -m fastmcp_extensions.utils.test_tool --app airbyte.mcp.server:app", help = "Test MCP tools directly with JSON arguments: poe mcp-tool-test <tool_name> '<json_args>'" }
+mcp-docs-md = { cmd = "python scripts/generate_mcp_markdown.py", help = "Generate Markdown docs for the MCP server into docs/mcp-generated/ (Docusaurus- and pdoc-compatible)" }
 
 # Claude Code MCP Testing Tasks
 [tool.poe.tasks.test-my-tools]