Skip to content

Simplify docstring pipeline: python emits the <Docstring> component directly #797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mishig25 merged 2 commits into from
Jul 4, 2026
Merged

Simplify docstring pipeline: python emits the <Docstring> component directly #797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e74731d
refactor: emit the <Docstring> svelte component directly from python
mishig25 Jul 4, 2026
9e281bd
fix(kit): silence state_referenced_locally warnings (runes follow-up)
mishig25 Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 105 additions & 129 deletions kit/preprocessors/docstring.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,50 @@ import htmlparser2 from "htmlparser2";
import { replaceAsync, renderSvelteChars, generateTagRegex } from "./utils.js";
import { mdsvexPreprocess } from "./mdsvex/index.js";

// Preprocessor that converts markdown into Docstring
// svelte component using mdsvexPreprocess
const REGEX_PARAMSDESC = generateTagRegex("paramsdesc");
const REGEX_PARAMSGROUP = generateTagRegex("paramsgroup", true);
const REGEX_PARAMSGROUP_TITLE = generateTagRegex("paramsgrouptitle");
const REGEX_PARAMSGROUP_DESC = generateTagRegex("paramsgroupdesc");
const REGEX_RETDESC = generateTagRegex("retdesc");
const REGEX_RETTYPE = generateTagRegex("rettype");
// note: fixes a long-standing tag mismatch (python emits `yielddesc`, the old regex
// matched `yieldesc`), which silently dropped yield descriptions
const REGEX_YIELDESC = generateTagRegex("yielddesc");
const REGEX_YIELDTYPE = generateTagRegex("yieldtype");
const REGEX_RAISEDESC = generateTagRegex("raises");
const REGEX_RAISETYPE = generateTagRegex("raisederrors");
const REGEX_TIP = /<Tip( warning={true})?>(((?!<Tip( warning={true})?>).)*)<\/Tip>/gms;
const REGEX_CHANGED =
/<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms;

/**
* Python (doc_builder/autodoc.py) emits the `<Docstring ...metadata props...>` component
* directly; its body carries the markdown-bearing sections (parameter descriptions,
* return/yield/raise types & descriptions), which must be rendered with the same mdsvex
* pipeline as the rest of the page. This preprocessor renders those sections into the
* remaining component props and closes the component.
*/
export const docstringPreprocess = {
markup: async ({ content, filename }) => {
const REGEX_DOCSTRING = generateTagRegex("docstring", true);
const REGEX_NAME = generateTagRegex("name");
const REGEX_ANCHOR = generateTagRegex("anchor");
const REGEX_SIGNATURE = generateTagRegex("parameters");
const REGEX_PARAMSDESC = generateTagRegex("paramsdesc");
const REGEX_PARAMSGROUPS = generateTagRegex("paramgroups");
const REGEX_RETDESC = generateTagRegex("retdesc");
const REGEX_RETTYPE = generateTagRegex("rettype");
const REGEX_YIELDESC = generateTagRegex("yieldesc");
const REGEX_YIELDTYPE = generateTagRegex("yieldtype");
const REGEX_RAISEDESC = generateTagRegex("raises");
const REGEX_RAISETYPE = generateTagRegex("raisederrors");
const REGEX_SOURCE = generateTagRegex("source");
const REGEX_TIP = /<Tip( warning={true})?>(((?!<Tip( warning={true})?>).)*)<\/Tip>/gms;
const REGEX_CHANGED =
/<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms;
const REGEX_IS_GETSET_DESC = /<isgetsetdescriptor>/ms;

content = await replaceAsync(content, REGEX_DOCSTRING, async (_, docstringBody) => {
docstringBody = renderSvelteChars(docstringBody);

const name = docstringBody.match(REGEX_NAME)[1];
const anchor = docstringBody.match(REGEX_ANCHOR)[1];
const signature = docstringBody.match(REGEX_SIGNATURE)[1];

let svelteComponent = `<Docstring name={${JSON.stringify(
unescapeUnderscores(name)
)}} anchor={${JSON.stringify(anchor)}} parameters={${signature}} `;

if (docstringBody.match(REGEX_PARAMSDESC)) {
let content = docstringBody.match(REGEX_PARAMSDESC)[1];
// The open tag ends at the first `>` followed by a newline: attribute values are
// single-line JSON, which may contain `>` inside strings but never a raw newline.
const REGEX_DOCSTRING = /<Docstring((?:(?!>\n)[\s\S])*)>\n([\s\S]*?)<\/Docstring>/g;

content = await replaceAsync(content, REGEX_DOCSTRING, async (_, metadataAttrs, body) => {
body = renderSvelteChars(body);

// parameter anchors are prefixed with the object's anchor
const anchor = metadataAttrs.match(/ anchor=\{"((?:[^"\\]|\\.)*)"\}/)?.[1] ?? "";

let svelteComponent = `<Docstring${metadataAttrs} `;

if (body.match(REGEX_PARAMSDESC)) {
let paramsContent = body.match(REGEX_PARAMSDESC)[1];
// escape }} by adding void character `&zwnj;` in between
content = content.replace(/}}/g, "}&zwnj;}");
let { code } = await mdsvexPreprocess.markup({ content, filename });
paramsContent = paramsContent.replace(/}}/g, "}&zwnj;}");
let { code } = await mdsvexPreprocess.markup({ content: paramsContent, filename });
// render <Tip> components that are inside parameter descriptions
code = code.replace(REGEX_TIP, (_, isWarning, tipContent) => {
code = code.replace(REGEX_TIP, (_tip, isWarning, tipContent) => {
const color = isWarning ? "orange" : "green";
return `<div
class="course-tip ${
Expand All @@ -53,133 +57,78 @@ export const docstringPreprocess = {
</div>`;
});
// render <Added>, <Changed>, <Deprecated> components that are inside parameter descriptions
code = code.replace(REGEX_CHANGED, (_, componentType, version, __, descriptionContent) => {
const color = /Added|Changed/.test(componentType) ? "green" : "orange";
if (!descriptionContent) {
descriptionContent = "";
}
return `<div
code = code.replace(
REGEX_CHANGED,
(_changed, componentType, version, __, descriptionContent) => {
const color = /Added|Changed/.test(componentType) ? "green" : "orange";
if (!descriptionContent) {
descriptionContent = "";
}
return `<div
class="course-tip ${
color === "orange" ? "course-tip-orange" : ""
} bg-gradient-to-br dark:bg-gradient-to-r before:border-${color}-500 dark:before:border-${color}-800 from-${color}-50 dark:from-gray-900 to-white dark:to-gray-950 border border-${color}-50 text-${color}-700 dark:text-gray-400"
>
<p class="font-medium">${componentType} in ${version}</p>
${descriptionContent}
</div>`;
});

const dom = htmlparser2.parseDocument(code);
const lists = domUtils.getElementsByTagName("ul", dom);
if (lists.length) {
const list = lists[0];
const result = [];
for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) {
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
const name = domUtils.innerText(nameEl);
const paramAnchor = `${anchor}.${name}`;
let description = domUtils.getInnerHTML(childEl).trim();

// strip enclosing paragraph tags <p> & </p>
if (description.startsWith("<p>")) {
description = description.slice("<p>".length);
}
if (description.endsWith("</p>")) {
description = description.slice(0, -"</p>".length);
}

result.push({ anchor: paramAnchor, description, name });
}
);

const result = parseRenderedParamsList(code, anchor);
if (result) {
svelteComponent += ` parametersDescription={${JSON.stringify(result)}} `;
}
}

if (docstringBody.match(REGEX_SOURCE)) {
const source = docstringBody.match(REGEX_SOURCE)[1];
svelteComponent += ` source={${JSON.stringify(source)}} `;
}

if (docstringBody.match(REGEX_RETDESC)) {
const retDesc = docstringBody.match(REGEX_RETDESC)[1];
if (body.match(REGEX_RETDESC)) {
const retDesc = body.match(REGEX_RETDESC)[1];
const { code } = await mdsvexPreprocess.markup({ content: retDesc, filename });
svelteComponent += ` returnDescription={${JSON.stringify(code)}} `;
}

if (docstringBody.match(REGEX_RETTYPE)) {
const retType = docstringBody.match(REGEX_RETTYPE)[1];
if (body.match(REGEX_RETTYPE)) {
const retType = body.match(REGEX_RETTYPE)[1];
const { code } = await mdsvexPreprocess.markup({ content: retType, filename });
svelteComponent += ` returnType={${JSON.stringify(code)}} `;
}

if (docstringBody.match(REGEX_YIELDESC)) {
const yieldDesc = docstringBody.match(REGEX_YIELDESC)[1];
if (body.match(REGEX_YIELDESC)) {
const yieldDesc = body.match(REGEX_YIELDESC)[1];
const { code } = await mdsvexPreprocess.markup({ content: yieldDesc, filename });
svelteComponent += ` returnDescription={${JSON.stringify(code)}} `;
}

if (docstringBody.match(REGEX_YIELDTYPE)) {
const yieldType = docstringBody.match(REGEX_YIELDTYPE)[1];
if (body.match(REGEX_YIELDTYPE)) {
const yieldType = body.match(REGEX_YIELDTYPE)[1];
const { code } = await mdsvexPreprocess.markup({ content: yieldType, filename });
svelteComponent += ` returnType={${JSON.stringify(code)}} isYield={true} `;
}

if (docstringBody.match(REGEX_RAISEDESC)) {
const raiseDesc = docstringBody.match(REGEX_RAISEDESC)[1];
if (body.match(REGEX_RAISEDESC)) {
const raiseDesc = body.match(REGEX_RAISEDESC)[1];
const { code } = await mdsvexPreprocess.markup({ content: raiseDesc, filename });
svelteComponent += ` raiseDescription={${JSON.stringify(code)}} `;
}

if (docstringBody.match(REGEX_RAISETYPE)) {
const raiseType = docstringBody.match(REGEX_RAISETYPE)[1];
if (body.match(REGEX_RAISETYPE)) {
const raiseType = body.match(REGEX_RAISETYPE)[1];
const { code } = await mdsvexPreprocess.markup({ content: raiseType, filename });
svelteComponent += ` raiseType={${JSON.stringify(code)}} `;
}

if (docstringBody.match(REGEX_IS_GETSET_DESC)) {
svelteComponent += ` isGetSetDescriptor={true} `;
const parameterGroups = [];
for (const [, groupBody] of body.matchAll(REGEX_PARAMSGROUP)) {
const title = groupBody.match(REGEX_PARAMSGROUP_TITLE)[1];
const groupContent = groupBody.match(REGEX_PARAMSGROUP_DESC)[1];
const { code } = await mdsvexPreprocess.markup({ content: groupContent, filename });
parameterGroups.push({
title,
parametersDescription: parseRenderedParamsList(code, anchor) ?? [],
});
}

if (docstringBody.match(REGEX_PARAMSGROUPS)) {
const nParamGroups = parseInt(docstringBody.match(REGEX_PARAMSGROUPS)[1]);
if (nParamGroups > 0) {
const parameterGroups = [];
for (let groupId = 1; groupId <= nParamGroups; groupId++) {
const REGEX_GROUP_TITLE = new RegExp(
`<paramsdesc${groupId}title>(((?!<paramsdesc${groupId}title>).)*)</paramsdesc${groupId}title>`,
"ms"
);
const REGEX_GROUP_CONTENT = new RegExp(
`<paramsdesc${groupId}>(((?!<paramsdesc${groupId}>).)*)</paramsdesc${groupId}>`,
"ms"
);
const title = docstringBody.match(REGEX_GROUP_TITLE)[1];
const content = docstringBody.match(REGEX_GROUP_CONTENT)[1];
const { code } = await mdsvexPreprocess.markup({ content, filename });
const dom = htmlparser2.parseDocument(code);
const lists = domUtils.getElementsByTagName("ul", dom);
const result = [];
if (lists.length) {
const list = lists[0];
for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) {
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
const name = domUtils.innerText(nameEl);
const paramAnchor = `${anchor}.${name}`;
let description = domUtils.getInnerHTML(childEl).trim();

// strip enclosing paragraph tags <p> & </p>
if (description.startsWith("<p>")) {
description = description.slice("<p>".length);
}
if (description.endsWith("</p>")) {
description = description.slice(0, -"</p>".length);
}

result.push({ anchor: paramAnchor, description, name });
}
}
parameterGroups.push({ title, parametersDescription: result });
}
svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `;
}
if (parameterGroups.length) {
svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `;
}

svelteComponent += ` />\n`;
Expand All @@ -191,8 +140,35 @@ export const docstringPreprocess = {
};

/**
* The mdx file contains unnecessarily escaped underscores in the docstring's name
* The parameter descriptions are authored as a markdown bullet list (`- **name** -- desc`)
* and must be rendered as one list to keep exact markdown semantics (tight/loose lists);
* this parses the rendered `<ul>` back into per-parameter structured data for the
* `Docstring` component (tooltips, per-parameter anchors).
* @param {string} code rendered html
* @param {string} anchor the object's anchor, used as prefix for parameter anchors
*/
function unescapeUnderscores(content) {
return content.replace(/\\_/g, "_");
function parseRenderedParamsList(code, anchor) {
const dom = htmlparser2.parseDocument(code);
const lists = domUtils.getElementsByTagName("ul", dom);
if (!lists.length) {
return undefined;
}
const result = [];
for (const childEl of lists[0].childNodes.filter(({ type }) => type === "tag")) {
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
const name = domUtils.innerText(nameEl);
const paramAnchor = `${anchor}.${name}`;
let description = domUtils.getInnerHTML(childEl).trim();

// strip enclosing paragraph tags <p> & </p>
if (description.startsWith("<p>")) {
description = description.slice("<p>".length);
}
if (description.endsWith("</p>")) {
description = description.slice(0, -"</p>".length);
}

result.push({ anchor: paramAnchor, description, name });
}
return result;
}
5 changes: 4 additions & 1 deletion kit/svelte.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ const config = {
warning.code?.startsWith("a11y") ||
warning.message.includes("A11y") ||
// md-generated doc content is full of `<video ... />` style self-closing tags
warning.code === "element_invalid_self_closing_tag"
warning.code === "element_invalid_self_closing_tag" ||
// initial-value capture of props is intentional here: doc pages pass
// static props (same semantics as the pre-runes svelte 4 code)
warning.code === "state_referenced_locally"
) {
/// Too noisy
return;
Expand Down
41 changes: 24 additions & 17 deletions src/doc_builder/autodoc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,29 +194,36 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l
object_doc = remove_example_tags(object_doc)
object_doc = hashlink_example_codeblock(object_doc, anchor)

svelte_str = "<docstring>"
svelte_str += f"<name>{name}</name>"
svelte_str += f"<anchor>{anchor}</anchor>"
# Metadata props are emitted directly as svelte component attributes. Their values
# are JS strings/JSON, where `{`, `<` and `#` are safe, so the MDX escaping
# (see `convert_special_chars`) is undone; `\_` needs no markdown escaping either.
def prop_value(text):
return text.replace("&amp;lcub;", "{").replace("&amp;lt;", "<").replace("&amp;num;", "#")

name = prop_value(str(name)).replace("\\_", "_")
svelte_str = f"<Docstring name={{{json.dumps(name)}}}"
svelte_str += f" anchor={{{json.dumps(prop_value(str(anchor)))}}}"
if source_link:
svelte_str += f"<source>{source_link}</source>"
svelte_str += f"<parameters>{json.dumps(signature)}</parameters>"
svelte_str += f" source={{{json.dumps(prop_value(source_link))}}}"
svelte_str += f" parameters={{{prop_value(json.dumps(signature))}}}"
if is_getset_desc:
svelte_str += "<isgetsetdescriptor>"
svelte_str += " isGetSetDescriptor={true}"
# the open tag must end with `>\n`: attribute values are single-line JSON, so the
# kit preprocessor (kit/preprocessors/docstring.js) can find the end of the tag
# unambiguously. The body carries the markdown-bearing sections, which the kit
# preprocessor renders (mdsvex) into the remaining props.
svelte_str += ">\n"

if parameters is not None:
parameters_str = ""
groups = _re_parameter_group.split(parameters)
group_default = groups.pop(0)
parameters_str += f"<paramsdesc>{group_default}</paramsdesc>"
n_groups = len(groups) // 2
for idx in range(n_groups):
id = idx + 1
svelte_str += f"<paramsdesc>{group_default}</paramsdesc>"
for idx in range(len(groups) // 2):
title, group = groups[2 * idx], groups[2 * idx + 1]
parameters_str += f"<paramsdesc{id}title>{title}</paramsdesc{id}title>"
parameters_str += f"<paramsdesc{id}>{group}</paramsdesc{id}>"

svelte_str += parameters_str
svelte_str += f"<paramgroups>{n_groups}</paramgroups>"
svelte_str += "<paramsgroup>"
svelte_str += f"<paramsgrouptitle>{title}</paramsgrouptitle>"
svelte_str += f"<paramsgroupdesc>{group}</paramsgroupdesc>"
svelte_str += "</paramsgroup>"

if returntype is not None:
svelte_str += f"<rettype>{returntype}</rettype>"
Expand All @@ -233,7 +240,7 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l
if raisederrors is not None:
svelte_str += f"<raisederrors>{raisederrors}</raisederrors>"

svelte_str += "</docstring>"
svelte_str += "</Docstring>"

return svelte_str + f"\n{object_doc}\n"

Expand Down
Loading