diff --git a/src/doc_builder/utils.py b/src/doc_builder/utils.py index 255ad3b2..6c81aa6f 100644 --- a/src/doc_builder/utils.py +++ b/src/doc_builder/utils.py @@ -14,6 +14,7 @@ import importlib.machinery import importlib.util +import json import os import re import shutil @@ -375,61 +376,98 @@ def strip_html_tags(text: str) -> str: return stripper.get_data() +def _extract_svelte_prop(props: str, prop_name: str): + """ + Read the JSON value of a `name={}` svelte attribute out of an opening-tag + attribute string, or return `None` if the prop is absent or malformed. + + The JSON is parsed with `raw_decode` so values containing `}` or `>` (type + annotations, defaults, ...) don't confuse the extraction. + """ + match = re.search(rf"\b{prop_name}=\{{", props) + if match is None: + return None + try: + value, end = json.JSONDecoder().raw_decode(props, match.end()) + except ValueError: + return None + if not props[end:].lstrip().startswith("}"): + return None + return value + + +def _extract_section(block: str, tag: str): + """Return the (stripped) content of `...` in `block`, or `None`.""" + match = re.search(rf"<{tag}>(.*?)", block, re.DOTALL) + return match.group(1).strip() if match else None + + def extract_docstring_info(docstring_block: str) -> dict: - """Extract information from a docstring block.""" + """ + Extract information from a `...` block. + + Metadata (name, anchor, source, signature) comes from the component props, the + markdown-bearing sections (parameters, returns, ...) from the component body. + See `doc_builder.autodoc.get_signature_component_svelte` for the emitted shape. + """ info = { "name": None, "anchor": None, "source": None, "parameters": None, "paramsdesc": None, + "paramsgroups": [], "rettype": None, "retdesc": None, - "description": None, + "yieldtype": None, + "yielddesc": None, + "raisederrors": None, + "raises": None, + "is_getset_descriptor": False, } - # Extract name - name_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if name_match: - raw_name = name_match.group(1).strip() + open_tag = re.match(r"[^\n]*)>", docstring_block) + props = open_tag.group("props") if open_tag else "" + + name = _extract_svelte_prop(props, "name") + if name: # Remove "class " or "def " prefix if present - cleaned_name = re.sub(r"^(class|def)\s+", "", raw_name) - info["name"] = cleaned_name - - # Extract anchor - anchor_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if anchor_match: - info["anchor"] = anchor_match.group(1).strip() - - # Extract source - source_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if source_match: - info["source"] = source_match.group(1).strip() - - # Extract parameters description - paramsdesc_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if paramsdesc_match: - info["paramsdesc"] = paramsdesc_match.group(1).strip() - - # Extract return type - rettype_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if rettype_match: - info["rettype"] = rettype_match.group(1).strip() - - # Extract return description - retdesc_match = re.search(r"(.*?)", docstring_block, re.DOTALL) - if retdesc_match: - info["retdesc"] = retdesc_match.group(1).strip() - - # Extract text outside docstring tags but inside the div - # This is the description text - description_match = re.search(r"(.*?)(?:|$)", docstring_block, re.DOTALL) - if description_match: - desc_text = description_match.group(1).strip() - # Remove any remaining HTML tags - desc_text = re.sub(r"<[^>]+>", "", desc_text) - if desc_text: - info["description"] = desc_text + info["name"] = re.sub(r"^(class|def)\s+", "", name.strip()) + + anchor = _extract_svelte_prop(props, "anchor") + # `anchor` is stringified python-side, so a missing anchor arrives as `"None"`. + if anchor and anchor != "None": + info["anchor"] = anchor.strip() + + source = _extract_svelte_prop(props, "source") + if source: + info["source"] = source.strip() + + parameters = _extract_svelte_prop(props, "parameters") + if isinstance(parameters, list): + info["parameters"] = parameters + + info["is_getset_descriptor"] = bool(_extract_svelte_prop(props, "isGetSetDescriptor")) + + for key, tag in ( + ("paramsdesc", "paramsdesc"), + ("rettype", "rettype"), + ("retdesc", "retdesc"), + ("yieldtype", "yieldtype"), + ("yielddesc", "yielddesc"), + ("raisederrors", "raisederrors"), + ("raises", "raises"), + ): + info[key] = _extract_section(docstring_block, tag) + + # Extra parameter groups, e.g. transformers' "Parameters for sequence generation". + for group in re.findall(r"(.*?)", docstring_block, re.DOTALL): + info["paramsgroups"].append( + { + "title": _extract_section(group, "paramsgrouptitle"), + "desc": _extract_section(group, "paramsgroupdesc"), + } + ) return info @@ -476,6 +514,23 @@ def format_parameters(paramsdesc: str) -> str: return "\n".join(formatted_params) +def format_call_signature(name: str, parameters) -> str: + """ + Render the `parameters` prop (a list of `{"name": ..., "val": ...}`) as a call + signature, e.g. ``HfApi.merge_pull_request(discussion_num: int, token = None)``. + """ + args = ", ".join(f"{param.get('name', '')}{param.get('val', '')}".strip() for param in parameters) + return f"{name}({args})" + + +def format_type(value: str) -> str: + """ + Wrap a return/yield/raise type in backticks, unless it already carries markup + (a resolved `[Name](url)` doc link, inline code, ...) that backticks would break. + """ + return value if re.search(r"[`\[\]<>]", value) else f"`{value}`" + + def process_docstring_block(docstring_block: str) -> str: """ Process a docstring block by: @@ -497,6 +552,13 @@ def process_docstring_block(docstring_block: str) -> str: parts.append(f"#### {info['name']}") parts.append("") + # Add the call signature (properties are not callable, so they have none) + if info["parameters"] is not None and not info["is_getset_descriptor"]: + parts.append("```python") + parts.append(format_call_signature(info["name"], info["parameters"])) + parts.append("```") + parts.append("") + # Add source link if available if info["source"]: # Strip any HTML from source @@ -504,11 +566,6 @@ def process_docstring_block(docstring_block: str) -> str: parts.append(f"[Source]({source_clean})") parts.append("") - # Add description - if info["description"]: - parts.append(info["description"]) - parts.append("") - # Add parameters description if info["paramsdesc"]: parts.append("**Parameters:**") @@ -518,22 +575,31 @@ def process_docstring_block(docstring_block: str) -> str: parts.append(formatted_params) parts.append("") - # Add return type - if info["rettype"]: - parts.append("**Returns:**") + # Add the extra parameter groups, if any + for group in info["paramsgroups"]: + if not group["desc"]: + continue + parts.append(f"**{group['title'] or 'Parameters'}:**") parts.append("") - # Strip HTML tags from return type - rettype_clean = strip_html_tags(info["rettype"]) - parts.append(f"`{rettype_clean}`") + parts.append(format_parameters(group["desc"])) parts.append("") - # Add return description - if info["retdesc"]: - if not info["rettype"]: - parts.append("**Returns:**") - parts.append("") - parts.append(info["retdesc"]) + # Add the returns / yields / raises sections + for type_key, desc_key, label in ( + ("rettype", "retdesc", "Returns"), + ("yieldtype", "yielddesc", "Yields"), + ("raisederrors", "raises", "Raises"), + ): + if not info[type_key] and not info[desc_key]: + continue + header = f"**{label}:**" + if info[type_key]: + header += f" {format_type(info[type_key])}" + parts.append(header) parts.append("") + if info[desc_key]: + parts.append(info[desc_key]) + parts.append("") result = "\n".join(parts) @@ -611,18 +677,23 @@ def strip_html_from_markdown(content: str) -> str: Strip HTML from markdown content. Handles: - - Docstring blocks wrapped in
...
+ - `...` components emitted by autodoc, which become a + level-4 heading with the signature, then the `**Parameters:**`/`**Returns:**` + sections. The object description follows the component in the source, so it + keeps its place right after those (same order as the rendered HTML page). - Other HTML tags throughout the document """ result = content - # Process docstring blocks with their wrapping divs - # Pattern to match:
.........
- docstring_pattern = r']*class="docstring[^"]*"[^>]*>.*?.*?.*?' + # The opening tag's attribute values are single-line JSON (see + # `get_signature_component_svelte`), so it ends at the last `>` on its own line. + docstring_pattern = r"\n.*?" def replace_docstring(match): block = match.group(0) - return process_docstring_block(block) + # Trailing newline: the object description directly follows the component (with a + # single newline in between), and must not end up glued to the last section. + return process_docstring_block(block) + "\n" result = re.sub(docstring_pattern, replace_docstring, result, flags=re.DOTALL) diff --git a/tests/test_utils.py b/tests/test_utils.py index 15a030fd..742ab387 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -127,3 +127,82 @@ def test_strip_html_still_removes_real_tags(self): self.assertNotIn("` markup instead of the `` props. + content = ( + '
\n\n' + '\n' + "- **endpoint** (`str`, *optional*) --\n" + " Endpoint of the Hub.\n" + "Client to interact with the Hugging Face Hub via HTTP.\n\n" + '
\n\n' + '\n' + "- **repo_id** (`str`) --\n" + " A namespace and a repo name separated\n" + " by a `/`.\n" + "- **discussion_num** (`int`) --\n" + " The number of the Pull Request." + "[DiscussionStatusChange](https://hf.co/docs#DiscussionStatusChange)" + "the status change event\n" + "Merges a Pull Request.\n\n" + "
\n" + ) + result = strip_html_from_markdown(content) + + # Class: heading with anchor, signature and parameters + self.assertIn("#### huggingface_hub.HfApi[[huggingface_hub.HfApi]]", result) + self.assertIn("huggingface_hub.HfApi(endpoint: str | None = None)", result) + self.assertIn( + "[Source](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/hf_api.py#L2226)", + result, + ) + # Method: heading with anchor, signature, and both section labels + self.assertIn("#### merge_pull_request[[huggingface_hub.HfApi.merge_pull_request]]", result) + self.assertIn("merge_pull_request(repo_id: str, discussion_num: int)", result) + self.assertEqual(result.count("**Parameters:**"), 2) + # A resolved doc link as return type keeps its markdown (no backtick wrapping) + self.assertIn("**Returns:** [DiscussionStatusChange](https://hf.co/docs#DiscussionStatusChange)", result) + # The description must not be glued to the last section of the docstring block + self.assertIn("the status change event\n\nMerges a Pull Request.", result) + # No leftover component markup + self.assertNotIn("\n\n' + '\n' + "\n" + "Get the content of this `AddedToken`\n\n" + "\n" + ) + result = strip_html_from_markdown(content) + self.assertIn("#### content", result) + self.assertNotIn("[[None]]", result) + self.assertNotIn("content()", result) + self.assertIn("Get the content of this `AddedToken`", result) + + # ... whereas a parameterless method keeps its (empty) signature + content = '\n\nResets it.\n' + self.assertIn("dummy.reset()", strip_html_from_markdown(content)) + + def test_strip_html_from_markdown_docstring_props_with_angle_brackets(self): + # `>` inside the props JSON must not truncate the opening tag. + content = ( + '"}]}>\n' + "`int`\n" + "Does something.\n" + ) + result = strip_html_from_markdown(content) + self.assertIn("#### dummy.func[[dummy.func]]", result) + self.assertIn("dummy.func(cb: Callable[[int], int] = )", result) + self.assertIn("**Returns:** `int`", result) + self.assertIn("Does something.", result)