Fix .md export losing method names, signatures and section labels (#810)

`strip_html_from_markdown` still looked for the legacy
`<div class="docstring">...<docstring>...</docstring>` markup, which
`autodoc` stopped emitting in #797 (it now emits the `<Docstring>`
component with metadata as props). Nothing matched, so the whole opening
tag was deleted by the generic tag cleanup -- taking the name, anchor and
signature with it -- and the `**Parameters:**` / `**Returns:**` labels
vanished too, gluing the return type onto the last parameter line.

Parse the current form instead: props from the opening tag (whose
attribute values are single-line JSON, decoded with `raw_decode` so `}`
and `>` inside type annotations can't truncate it), sections from the
component body. The signature is rendered from the `parameters` prop, and
the parameter groups / yields / raises sections -- previously dropped
entirely -- are emitted as well. Types are only wrapped in backticks when
they carry no markup, since that would break a resolved `[Name](url)` doc
link.

Section order (signature, parameters, returns, then the description that
follows the component) matches how `Docstring.svelte` renders the page.

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: mishig25

src/doc_builder/utils.py

@@ -14,6 +14,7 @@
 
 import importlib.machinery
 import importlib.util
+import json
 import os
 import re
 import shutil
@@ -375,61 +376,98 @@ def strip_html_tags(text: str) -> str:
     return stripper.get_data()
 
 
+def _extract_svelte_prop(props: str, prop_name: str):
+    """
+    Read the JSON value of a `name={<json>}` svelte attribute out of an opening-tag
+    attribute string, or return `None` if the prop is absent or malformed.
+
+    The JSON is parsed with `raw_decode` so values containing `}` or `>` (type
+    annotations, defaults, ...) don't confuse the extraction.
+    """
+    match = re.search(rf"\b{prop_name}=\{{", props)
+    if match is None:
+        return None
+    try:
+        value, end = json.JSONDecoder().raw_decode(props, match.end())
+    except ValueError:
+        return None
+    if not props[end:].lstrip().startswith("}"):
+        return None
+    return value
+
+
+def _extract_section(block: str, tag: str):
+    """Return the (stripped) content of `<tag>...</tag>` in `block`, or `None`."""
+    match = re.search(rf"<{tag}>(.*?)</{tag}>", block, re.DOTALL)
+    return match.group(1).strip() if match else None
+
+
 def extract_docstring_info(docstring_block: str) -> dict:
-    """Extract information from a docstring block."""
+    """
+    Extract information from a `<Docstring ...>...</Docstring>` block.
+
+    Metadata (name, anchor, source, signature) comes from the component props, the
+    markdown-bearing sections (parameters, returns, ...) from the component body.
+    See `doc_builder.autodoc.get_signature_component_svelte` for the emitted shape.
+    """
     info = {
         "name": None,
         "anchor": None,
         "source": None,
         "parameters": None,
         "paramsdesc": None,
+        "paramsgroups": [],
         "rettype": None,
         "retdesc": None,
-        "description": None,
+        "yieldtype": None,
+        "yielddesc": None,
+        "raisederrors": None,
+        "raises": None,
+        "is_getset_descriptor": False,
     }
 
-    # Extract name
-    name_match = re.search(r"<name>(.*?)</name>", docstring_block, re.DOTALL)
-    if name_match:
-        raw_name = name_match.group(1).strip()
+    open_tag = re.match(r"<Docstring\s(?P<props>[^\n]*)>", docstring_block)
+    props = open_tag.group("props") if open_tag else ""
+
+    name = _extract_svelte_prop(props, "name")
+    if name:
         # Remove "class " or "def " prefix if present
-        cleaned_name = re.sub(r"^(class|def)\s+", "", raw_name)
-        info["name"] = cleaned_name
-
-    # Extract anchor
-    anchor_match = re.search(r"<anchor>(.*?)</anchor>", docstring_block, re.DOTALL)
-    if anchor_match:
-        info["anchor"] = anchor_match.group(1).strip()
-
-    # Extract source
-    source_match = re.search(r"<source>(.*?)</source>", docstring_block, re.DOTALL)
-    if source_match:
-        info["source"] = source_match.group(1).strip()
-
-    # Extract parameters description
-    paramsdesc_match = re.search(r"<paramsdesc>(.*?)</paramsdesc>", docstring_block, re.DOTALL)
-    if paramsdesc_match:
-        info["paramsdesc"] = paramsdesc_match.group(1).strip()
-
-    # Extract return type
-    rettype_match = re.search(r"<rettype>(.*?)</rettype>", docstring_block, re.DOTALL)
-    if rettype_match:
-        info["rettype"] = rettype_match.group(1).strip()
-
-    # Extract return description
-    retdesc_match = re.search(r"<retdesc>(.*?)</retdesc>", docstring_block, re.DOTALL)
-    if retdesc_match:
-        info["retdesc"] = retdesc_match.group(1).strip()
-
-    # Extract text outside docstring tags but inside the div
-    # This is the description text
-    description_match = re.search(r"</docstring>(.*?)(?:</div>|$)", docstring_block, re.DOTALL)
-    if description_match:
-        desc_text = description_match.group(1).strip()
-        # Remove any remaining HTML tags
-        desc_text = re.sub(r"<[^>]+>", "", desc_text)
-        if desc_text:
-            info["description"] = desc_text
+        info["name"] = re.sub(r"^(class|def)\s+", "", name.strip())
+
+    anchor = _extract_svelte_prop(props, "anchor")
+    # `anchor` is stringified python-side, so a missing anchor arrives as `"None"`.
+    if anchor and anchor != "None":
+        info["anchor"] = anchor.strip()
+
+    source = _extract_svelte_prop(props, "source")
+    if source:
+        info["source"] = source.strip()
+
+    parameters = _extract_svelte_prop(props, "parameters")
+    if isinstance(parameters, list):
+        info["parameters"] = parameters
+
+    info["is_getset_descriptor"] = bool(_extract_svelte_prop(props, "isGetSetDescriptor"))
+
+    for key, tag in (
+        ("paramsdesc", "paramsdesc"),
+        ("rettype", "rettype"),
+        ("retdesc", "retdesc"),
+        ("yieldtype", "yieldtype"),
+        ("yielddesc", "yielddesc"),
+        ("raisederrors", "raisederrors"),
+        ("raises", "raises"),
+    ):
+        info[key] = _extract_section(docstring_block, tag)
+
+    # Extra parameter groups, e.g. transformers' "Parameters for sequence generation".
+    for group in re.findall(r"<paramsgroup>(.*?)</paramsgroup>", docstring_block, re.DOTALL):
+        info["paramsgroups"].append(
+            {
+                "title": _extract_section(group, "paramsgrouptitle"),
+                "desc": _extract_section(group, "paramsgroupdesc"),
+            }
+        )
 
     return info
 
@@ -476,6 +514,23 @@ def format_parameters(paramsdesc: str) -> str:
     return "\n".join(formatted_params)
 
 
+def format_call_signature(name: str, parameters) -> str:
+    """
+    Render the `parameters` prop (a list of `{"name": ..., "val": ...}`) as a call
+    signature, e.g. ``HfApi.merge_pull_request(discussion_num: int, token = None)``.
+    """
+    args = ", ".join(f"{param.get('name', '')}{param.get('val', '')}".strip() for param in parameters)
+    return f"{name}({args})"
+
+
+def format_type(value: str) -> str:
+    """
+    Wrap a return/yield/raise type in backticks, unless it already carries markup
+    (a resolved `[Name](url)` doc link, inline code, ...) that backticks would break.
+    """
+    return value if re.search(r"[`\[\]<>]", value) else f"`{value}`"
+
+
 def process_docstring_block(docstring_block: str) -> str:
     """
     Process a docstring block by:
@@ -497,18 +552,20 @@ def process_docstring_block(docstring_block: str) -> str:
             parts.append(f"#### {info['name']}")
         parts.append("")
 
+        # Add the call signature (properties are not callable, so they have none)
+        if info["parameters"] is not None and not info["is_getset_descriptor"]:
+            parts.append("```python")
+            parts.append(format_call_signature(info["name"], info["parameters"]))
+            parts.append("```")
+            parts.append("")
+
     # Add source link if available
     if info["source"]:
         # Strip any HTML from source
         source_clean = strip_html_tags(info["source"])
         parts.append(f"[Source]({source_clean})")
         parts.append("")
 
-    # Add description
-    if info["description"]:
-        parts.append(info["description"])
-        parts.append("")
-
     # Add parameters description
     if info["paramsdesc"]:
         parts.append("**Parameters:**")
@@ -518,22 +575,31 @@ def process_docstring_block(docstring_block: str) -> str:
         parts.append(formatted_params)
         parts.append("")
 
-    # Add return type
-    if info["rettype"]:
-        parts.append("**Returns:**")
+    # Add the extra parameter groups, if any
+    for group in info["paramsgroups"]:
+        if not group["desc"]:
+            continue
+        parts.append(f"**{group['title'] or 'Parameters'}:**")
         parts.append("")
-        # Strip HTML tags from return type
-        rettype_clean = strip_html_tags(info["rettype"])
-        parts.append(f"`{rettype_clean}`")
+        parts.append(format_parameters(group["desc"]))
         parts.append("")
 
-    # Add return description
-    if info["retdesc"]:
-        if not info["rettype"]:
-            parts.append("**Returns:**")
-            parts.append("")
-        parts.append(info["retdesc"])
+    # Add the returns / yields / raises sections
+    for type_key, desc_key, label in (
+        ("rettype", "retdesc", "Returns"),
+        ("yieldtype", "yielddesc", "Yields"),
+        ("raisederrors", "raises", "Raises"),
+    ):
+        if not info[type_key] and not info[desc_key]:
+            continue
+        header = f"**{label}:**"
+        if info[type_key]:
+            header += f" {format_type(info[type_key])}"
+        parts.append(header)
         parts.append("")
+        if info[desc_key]:
+            parts.append(info[desc_key])
+            parts.append("")
 
     result = "\n".join(parts)
 
@@ -611,18 +677,23 @@ def strip_html_from_markdown(content: str) -> str:
     Strip HTML from markdown content.
 
     Handles:
-    - Docstring blocks wrapped in <div class="docstring...">...</div>
+    - `<Docstring ...>...</Docstring>` components emitted by autodoc, which become a
+      level-4 heading with the signature, then the `**Parameters:**`/`**Returns:**`
+      sections. The object description follows the component in the source, so it
+      keeps its place right after those (same order as the rendered HTML page).
     - Other HTML tags throughout the document
     """
     result = content
 
-    # Process docstring blocks with their wrapping divs
-    # Pattern to match: <div class="docstring...">...<docstring>...</docstring>...</div>
-    docstring_pattern = r'<div[^>]*class="docstring[^"]*"[^>]*>.*?<docstring>.*?</docstring>.*?</div>'
+    # The opening tag's attribute values are single-line JSON (see
+    # `get_signature_component_svelte`), so it ends at the last `>` on its own line.
+    docstring_pattern = r"<Docstring\s[^\n]*>\n.*?</Docstring>"
 
     def replace_docstring(match):
         block = match.group(0)
-        return process_docstring_block(block)
+        # Trailing newline: the object description directly follows the component (with a
+        # single newline in between), and must not end up glued to the last section.
+        return process_docstring_block(block) + "\n"
 
     result = re.sub(docstring_pattern, replace_docstring, result, flags=re.DOTALL)
 

tests/test_utils.py

@@ -127,3 +127,82 @@ def test_strip_html_still_removes_real_tags(self):
         self.assertNotIn("<div", result)
         self.assertIn("warning", result)
         self.assertIn("inside", result)
+
+    def test_strip_html_from_markdown_docstring_component(self):
+        # Regression: method names, anchors, signatures and the section labels were
+        # dropped from the `.md` export because the old parser looked for the legacy
+        # `<docstring>` markup instead of the `<Docstring ...>` props.
+        content = (
+            '<div class="docstring border-l-2">\n\n'
+            '<Docstring name={"class huggingface_hub.HfApi"} anchor={"huggingface_hub.HfApi"} '
+            'source={"https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/hf_api.py#L2226"} '
+            'parameters={[{"name": "endpoint", "val": ": str | None = None"}]}>\n'
+            "<paramsdesc>- **endpoint** (`str`, *optional*) --\n"
+            "  Endpoint of the Hub.</paramsdesc></Docstring>\n"
+            "Client to interact with the Hugging Face Hub via HTTP.\n\n"
+            '<div class="docstring border-l-2">\n\n'
+            '<Docstring name={"merge_pull_request"} anchor={"huggingface_hub.HfApi.merge_pull_request"} '
+            'parameters={[{"name": "repo_id", "val": ": str"}, {"name": "discussion_num", "val": ": int"}]}>\n'
+            "<paramsdesc>- **repo_id** (`str`) --\n"
+            "  A namespace and a repo name separated\n"
+            "  by a `/`.\n"
+            "- **discussion_num** (`int`) --\n"
+            "  The number of the Pull Request.</paramsdesc>"
+            "<rettype>[DiscussionStatusChange](https://hf.co/docs#DiscussionStatusChange)</rettype>"
+            "<retdesc>the status change event</retdesc></Docstring>\n"
+            "Merges a Pull Request.\n\n"
+            "</div></div>\n"
+        )
+        result = strip_html_from_markdown(content)
+
+        # Class: heading with anchor, signature and parameters
+        self.assertIn("#### huggingface_hub.HfApi[[huggingface_hub.HfApi]]", result)
+        self.assertIn("huggingface_hub.HfApi(endpoint: str | None = None)", result)
+        self.assertIn(
+            "[Source](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/hf_api.py#L2226)",
+            result,
+        )
+        # Method: heading with anchor, signature, and both section labels
+        self.assertIn("#### merge_pull_request[[huggingface_hub.HfApi.merge_pull_request]]", result)
+        self.assertIn("merge_pull_request(repo_id: str, discussion_num: int)", result)
+        self.assertEqual(result.count("**Parameters:**"), 2)
+        # A resolved doc link as return type keeps its markdown (no backtick wrapping)
+        self.assertIn("**Returns:** [DiscussionStatusChange](https://hf.co/docs#DiscussionStatusChange)", result)
+        # The description must not be glued to the last section of the docstring block
+        self.assertIn("the status change event\n\nMerges a Pull Request.", result)
+        # No leftover component markup
+        self.assertNotIn("<Docstring", result)
+        self.assertNotIn("paramsdesc", result)
+
+    def test_strip_html_from_markdown_docstring_getset_descriptor(self):
+        # Properties have no parameters and no anchor: heading only, no empty signature.
+        content = (
+            '<div class="docstring border-l-2">\n\n'
+            '<Docstring name={"content"} anchor={"None"} parameters={[]} isGetSetDescriptor={true}>\n'
+            "</Docstring>\n"
+            "Get the content of this `AddedToken`\n\n"
+            "</div>\n"
+        )
+        result = strip_html_from_markdown(content)
+        self.assertIn("#### content", result)
+        self.assertNotIn("[[None]]", result)
+        self.assertNotIn("content()", result)
+        self.assertIn("Get the content of this `AddedToken`", result)
+
+        # ... whereas a parameterless method keeps its (empty) signature
+        content = '<Docstring name={"dummy.reset"} anchor={"dummy.reset"} parameters={[]}>\n</Docstring>\nResets it.\n'
+        self.assertIn("dummy.reset()", strip_html_from_markdown(content))
+
+    def test_strip_html_from_markdown_docstring_props_with_angle_brackets(self):
+        # `>` inside the props JSON must not truncate the opening tag.
+        content = (
+            '<Docstring name={"dummy.func"} anchor={"dummy.func"} '
+            'parameters={[{"name": "cb", "val": ": Callable[[int], int] = <factory>"}]}>\n'
+            "<rettype>`int`</rettype></Docstring>\n"
+            "Does something.\n"
+        )
+        result = strip_html_from_markdown(content)
+        self.assertIn("#### dummy.func[[dummy.func]]", result)
+        self.assertIn("dummy.func(cb: Callable[[int], int] = <factory>)", result)
+        self.assertIn("**Returns:** `int`", result)
+        self.assertIn("Does something.", result)