Skip to content

Commit 2f7376e

Browse files
mishig25claude
mishig25
and
claude
authored
Simplify docstring pipeline: python emits the <Docstring> component directly (#797)
* refactor: emit the <Docstring> svelte component directly from python autodoc.py previously serialized docstring metadata into a zoo of custom tags (<docstring><name>...<paramgroups>N) that kit/preprocessors/ docstring.js regex-parsed back into a <Docstring .../> component. Python now emits the component open tag with all metadata props inline (name/anchor/source/parameters/isGetSetDescriptor as final JSON values, MDX escaping undone since JS strings need none); the component body carries only the markdown-bearing sections, which the kit preprocessor renders with mdsvex into the remaining props and closes the component. - parameter groups become nested <paramsgroup> blocks instead of numbered tags + a count - the rendered-<ul> parsing stays (documented): parameter lists must be rendered as one markdown list to keep exact tight/loose list semantics - fixes a long-standing tag mismatch: python emits <yielddesc> but the old regex matched <yieldesc>, silently dropping yield descriptions Verified byte-identical output (48/48 pages, modulo hashed asset names and nondeterministic python object addresses) on the accelerate e2e build; python test suite updated and passing. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(kit): silence state_referenced_locally warnings (runes follow-up) This hunk was part of the runes migration verification but was left out of #795 (staged with `git add kit/src`, which misses kit/svelte.config.js). The flagged initial-value captures have the same semantics as the pre-runes svelte 4 code — doc pages pass static props — and the warning floods every downstream doc build log otherwise. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 7241b19 commit 2f7376e

4 files changed

Lines changed: 142 additions & 154 deletions

File tree

‎kit/preprocessors/docstring.js‎

Lines changed: 105 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -3,46 +3,50 @@ import htmlparser2 from "htmlparser2";
33
import { replaceAsync, renderSvelteChars, generateTagRegex } from "./utils.js";
44
import { mdsvexPreprocess } from "./mdsvex/index.js";
55

6-
// Preprocessor that converts markdown into Docstring
7-
// svelte component using mdsvexPreprocess
6+
const REGEX_PARAMSDESC = generateTagRegex("paramsdesc");
7+
const REGEX_PARAMSGROUP = generateTagRegex("paramsgroup", true);
8+
const REGEX_PARAMSGROUP_TITLE = generateTagRegex("paramsgrouptitle");
9+
const REGEX_PARAMSGROUP_DESC = generateTagRegex("paramsgroupdesc");
10+
const REGEX_RETDESC = generateTagRegex("retdesc");
11+
const REGEX_RETTYPE = generateTagRegex("rettype");
12+
// note: fixes a long-standing tag mismatch (python emits `yielddesc`, the old regex
13+
// matched `yieldesc`), which silently dropped yield descriptions
14+
const REGEX_YIELDESC = generateTagRegex("yielddesc");
15+
const REGEX_YIELDTYPE = generateTagRegex("yieldtype");
16+
const REGEX_RAISEDESC = generateTagRegex("raises");
17+
const REGEX_RAISETYPE = generateTagRegex("raisederrors");
18+
const REGEX_TIP = /<Tip( warning={true})?>(((?!<Tip( warning={true})?>).)*)<\/Tip>/gms;
19+
const REGEX_CHANGED =
20+
/<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms;
21+
22+
/**
23+
* Python (doc_builder/autodoc.py) emits the `<Docstring ...metadata props...>` component
24+
* directly; its body carries the markdown-bearing sections (parameter descriptions,
25+
* return/yield/raise types & descriptions), which must be rendered with the same mdsvex
26+
* pipeline as the rest of the page. This preprocessor renders those sections into the
27+
* remaining component props and closes the component.
28+
*/
829
export const docstringPreprocess = {
930
markup: async ({ content, filename }) => {
10-
const REGEX_DOCSTRING = generateTagRegex("docstring", true);
11-
const REGEX_NAME = generateTagRegex("name");
12-
const REGEX_ANCHOR = generateTagRegex("anchor");
13-
const REGEX_SIGNATURE = generateTagRegex("parameters");
14-
const REGEX_PARAMSDESC = generateTagRegex("paramsdesc");
15-
const REGEX_PARAMSGROUPS = generateTagRegex("paramgroups");
16-
const REGEX_RETDESC = generateTagRegex("retdesc");
17-
const REGEX_RETTYPE = generateTagRegex("rettype");
18-
const REGEX_YIELDESC = generateTagRegex("yieldesc");
19-
const REGEX_YIELDTYPE = generateTagRegex("yieldtype");
20-
const REGEX_RAISEDESC = generateTagRegex("raises");
21-
const REGEX_RAISETYPE = generateTagRegex("raisederrors");
22-
const REGEX_SOURCE = generateTagRegex("source");
23-
const REGEX_TIP = /<Tip( warning={true})?>(((?!<Tip( warning={true})?>).)*)<\/Tip>/gms;
24-
const REGEX_CHANGED =
25-
/<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms;
26-
const REGEX_IS_GETSET_DESC = /<isgetsetdescriptor>/ms;
27-
28-
content = await replaceAsync(content, REGEX_DOCSTRING, async (_, docstringBody) => {
29-
docstringBody = renderSvelteChars(docstringBody);
30-
31-
const name = docstringBody.match(REGEX_NAME)[1];
32-
const anchor = docstringBody.match(REGEX_ANCHOR)[1];
33-
const signature = docstringBody.match(REGEX_SIGNATURE)[1];
34-
35-
let svelteComponent = `<Docstring name={${JSON.stringify(
36-
unescapeUnderscores(name)
37-
)}} anchor={${JSON.stringify(anchor)}} parameters={${signature}} `;
38-
39-
if (docstringBody.match(REGEX_PARAMSDESC)) {
40-
let content = docstringBody.match(REGEX_PARAMSDESC)[1];
31+
// The open tag ends at the first `>` followed by a newline: attribute values are
32+
// single-line JSON, which may contain `>` inside strings but never a raw newline.
33+
const REGEX_DOCSTRING = /<Docstring((?:(?!>\n)[\s\S])*)>\n([\s\S]*?)<\/Docstring>/g;
34+
35+
content = await replaceAsync(content, REGEX_DOCSTRING, async (_, metadataAttrs, body) => {
36+
body = renderSvelteChars(body);
37+
38+
// parameter anchors are prefixed with the object's anchor
39+
const anchor = metadataAttrs.match(/ anchor=\{"((?:[^"\\]|\\.)*)"\}/)?.[1] ?? "";
40+
41+
let svelteComponent = `<Docstring${metadataAttrs} `;
42+
43+
if (body.match(REGEX_PARAMSDESC)) {
44+
let paramsContent = body.match(REGEX_PARAMSDESC)[1];
4145
// escape }} by adding void character `&zwnj;` in between
42-
content = content.replace(/}}/g, "}&zwnj;}");
43-
let { code } = await mdsvexPreprocess.markup({ content, filename });
46+
paramsContent = paramsContent.replace(/}}/g, "}&zwnj;}");
47+
let { code } = await mdsvexPreprocess.markup({ content: paramsContent, filename });
4448
// render <Tip> components that are inside parameter descriptions
45-
code = code.replace(REGEX_TIP, (_, isWarning, tipContent) => {
49+
code = code.replace(REGEX_TIP, (_tip, isWarning, tipContent) => {
4650
const color = isWarning ? "orange" : "green";
4751
return `<div
4852
class="course-tip ${
@@ -53,133 +57,78 @@ export const docstringPreprocess = {
5357
</div>`;
5458
});
5559
// render <Added>, <Changed>, <Deprecated> components that are inside parameter descriptions
56-
code = code.replace(REGEX_CHANGED, (_, componentType, version, __, descriptionContent) => {
57-
const color = /Added|Changed/.test(componentType) ? "green" : "orange";
58-
if (!descriptionContent) {
59-
descriptionContent = "";
60-
}
61-
return `<div
60+
code = code.replace(
61+
REGEX_CHANGED,
62+
(_changed, componentType, version, __, descriptionContent) => {
63+
const color = /Added|Changed/.test(componentType) ? "green" : "orange";
64+
if (!descriptionContent) {
65+
descriptionContent = "";
66+
}
67+
return `<div
6268
class="course-tip ${
6369
color === "orange" ? "course-tip-orange" : ""
6470
} bg-gradient-to-br dark:bg-gradient-to-r before:border-${color}-500 dark:before:border-${color}-800 from-${color}-50 dark:from-gray-900 to-white dark:to-gray-950 border border-${color}-50 text-${color}-700 dark:text-gray-400"
6571
>
6672
<p class="font-medium">${componentType} in ${version}</p>
6773
${descriptionContent}
6874
</div>`;
69-
});
70-
71-
const dom = htmlparser2.parseDocument(code);
72-
const lists = domUtils.getElementsByTagName("ul", dom);
73-
if (lists.length) {
74-
const list = lists[0];
75-
const result = [];
76-
for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) {
77-
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
78-
const name = domUtils.innerText(nameEl);
79-
const paramAnchor = `${anchor}.${name}`;
80-
let description = domUtils.getInnerHTML(childEl).trim();
81-
82-
// strip enclosing paragraph tags <p> & </p>
83-
if (description.startsWith("<p>")) {
84-
description = description.slice("<p>".length);
85-
}
86-
if (description.endsWith("</p>")) {
87-
description = description.slice(0, -"</p>".length);
88-
}
89-
90-
result.push({ anchor: paramAnchor, description, name });
9175
}
76+
);
77+
78+
const result = parseRenderedParamsList(code, anchor);
79+
if (result) {
9280
svelteComponent += ` parametersDescription={${JSON.stringify(result)}} `;
9381
}
9482
}
9583

96-
if (docstringBody.match(REGEX_SOURCE)) {
97-
const source = docstringBody.match(REGEX_SOURCE)[1];
98-
svelteComponent += ` source={${JSON.stringify(source)}} `;
99-
}
100-
101-
if (docstringBody.match(REGEX_RETDESC)) {
102-
const retDesc = docstringBody.match(REGEX_RETDESC)[1];
84+
if (body.match(REGEX_RETDESC)) {
85+
const retDesc = body.match(REGEX_RETDESC)[1];
10386
const { code } = await mdsvexPreprocess.markup({ content: retDesc, filename });
10487
svelteComponent += ` returnDescription={${JSON.stringify(code)}} `;
10588
}
10689

107-
if (docstringBody.match(REGEX_RETTYPE)) {
108-
const retType = docstringBody.match(REGEX_RETTYPE)[1];
90+
if (body.match(REGEX_RETTYPE)) {
91+
const retType = body.match(REGEX_RETTYPE)[1];
10992
const { code } = await mdsvexPreprocess.markup({ content: retType, filename });
11093
svelteComponent += ` returnType={${JSON.stringify(code)}} `;
11194
}
11295

113-
if (docstringBody.match(REGEX_YIELDESC)) {
114-
const yieldDesc = docstringBody.match(REGEX_YIELDESC)[1];
96+
if (body.match(REGEX_YIELDESC)) {
97+
const yieldDesc = body.match(REGEX_YIELDESC)[1];
11598
const { code } = await mdsvexPreprocess.markup({ content: yieldDesc, filename });
11699
svelteComponent += ` returnDescription={${JSON.stringify(code)}} `;
117100
}
118101

119-
if (docstringBody.match(REGEX_YIELDTYPE)) {
120-
const yieldType = docstringBody.match(REGEX_YIELDTYPE)[1];
102+
if (body.match(REGEX_YIELDTYPE)) {
103+
const yieldType = body.match(REGEX_YIELDTYPE)[1];
121104
const { code } = await mdsvexPreprocess.markup({ content: yieldType, filename });
122105
svelteComponent += ` returnType={${JSON.stringify(code)}} isYield={true} `;
123106
}
124107

125-
if (docstringBody.match(REGEX_RAISEDESC)) {
126-
const raiseDesc = docstringBody.match(REGEX_RAISEDESC)[1];
108+
if (body.match(REGEX_RAISEDESC)) {
109+
const raiseDesc = body.match(REGEX_RAISEDESC)[1];
127110
const { code } = await mdsvexPreprocess.markup({ content: raiseDesc, filename });
128111
svelteComponent += ` raiseDescription={${JSON.stringify(code)}} `;
129112
}
130113

131-
if (docstringBody.match(REGEX_RAISETYPE)) {
132-
const raiseType = docstringBody.match(REGEX_RAISETYPE)[1];
114+
if (body.match(REGEX_RAISETYPE)) {
115+
const raiseType = body.match(REGEX_RAISETYPE)[1];
133116
const { code } = await mdsvexPreprocess.markup({ content: raiseType, filename });
134117
svelteComponent += ` raiseType={${JSON.stringify(code)}} `;
135118
}
136119

137-
if (docstringBody.match(REGEX_IS_GETSET_DESC)) {
138-
svelteComponent += ` isGetSetDescriptor={true} `;
120+
const parameterGroups = [];
121+
for (const [, groupBody] of body.matchAll(REGEX_PARAMSGROUP)) {
122+
const title = groupBody.match(REGEX_PARAMSGROUP_TITLE)[1];
123+
const groupContent = groupBody.match(REGEX_PARAMSGROUP_DESC)[1];
124+
const { code } = await mdsvexPreprocess.markup({ content: groupContent, filename });
125+
parameterGroups.push({
126+
title,
127+
parametersDescription: parseRenderedParamsList(code, anchor) ?? [],
128+
});
139129
}
140-
141-
if (docstringBody.match(REGEX_PARAMSGROUPS)) {
142-
const nParamGroups = parseInt(docstringBody.match(REGEX_PARAMSGROUPS)[1]);
143-
if (nParamGroups > 0) {
144-
const parameterGroups = [];
145-
for (let groupId = 1; groupId <= nParamGroups; groupId++) {
146-
const REGEX_GROUP_TITLE = new RegExp(
147-
`<paramsdesc${groupId}title>(((?!<paramsdesc${groupId}title>).)*)</paramsdesc${groupId}title>`,
148-
"ms"
149-
);
150-
const REGEX_GROUP_CONTENT = new RegExp(
151-
`<paramsdesc${groupId}>(((?!<paramsdesc${groupId}>).)*)</paramsdesc${groupId}>`,
152-
"ms"
153-
);
154-
const title = docstringBody.match(REGEX_GROUP_TITLE)[1];
155-
const content = docstringBody.match(REGEX_GROUP_CONTENT)[1];
156-
const { code } = await mdsvexPreprocess.markup({ content, filename });
157-
const dom = htmlparser2.parseDocument(code);
158-
const lists = domUtils.getElementsByTagName("ul", dom);
159-
const result = [];
160-
if (lists.length) {
161-
const list = lists[0];
162-
for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) {
163-
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
164-
const name = domUtils.innerText(nameEl);
165-
const paramAnchor = `${anchor}.${name}`;
166-
let description = domUtils.getInnerHTML(childEl).trim();
167-
168-
// strip enclosing paragraph tags <p> & </p>
169-
if (description.startsWith("<p>")) {
170-
description = description.slice("<p>".length);
171-
}
172-
if (description.endsWith("</p>")) {
173-
description = description.slice(0, -"</p>".length);
174-
}
175-
176-
result.push({ anchor: paramAnchor, description, name });
177-
}
178-
}
179-
parameterGroups.push({ title, parametersDescription: result });
180-
}
181-
svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `;
182-
}
130+
if (parameterGroups.length) {
131+
svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `;
183132
}
184133

185134
svelteComponent += ` />\n`;
@@ -191,8 +140,35 @@ export const docstringPreprocess = {
191140
};
192141

193142
/**
194-
* The mdx file contains unnecessarily escaped underscores in the docstring's name
143+
* The parameter descriptions are authored as a markdown bullet list (`- **name** -- desc`)
144+
* and must be rendered as one list to keep exact markdown semantics (tight/loose lists);
145+
* this parses the rendered `<ul>` back into per-parameter structured data for the
146+
* `Docstring` component (tooltips, per-parameter anchors).
147+
* @param {string} code rendered html
148+
* @param {string} anchor the object's anchor, used as prefix for parameter anchors
195149
*/
196-
function unescapeUnderscores(content) {
197-
return content.replace(/\\_/g, "_");
150+
function parseRenderedParamsList(code, anchor) {
151+
const dom = htmlparser2.parseDocument(code);
152+
const lists = domUtils.getElementsByTagName("ul", dom);
153+
if (!lists.length) {
154+
return undefined;
155+
}
156+
const result = [];
157+
for (const childEl of lists[0].childNodes.filter(({ type }) => type === "tag")) {
158+
const nameEl = domUtils.getElementsByTagName("strong", childEl)[0];
159+
const name = domUtils.innerText(nameEl);
160+
const paramAnchor = `${anchor}.${name}`;
161+
let description = domUtils.getInnerHTML(childEl).trim();
162+
163+
// strip enclosing paragraph tags <p> & </p>
164+
if (description.startsWith("<p>")) {
165+
description = description.slice("<p>".length);
166+
}
167+
if (description.endsWith("</p>")) {
168+
description = description.slice(0, -"</p>".length);
169+
}
170+
171+
result.push({ anchor: paramAnchor, description, name });
172+
}
173+
return result;
198174
}

‎kit/svelte.config.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,10 @@ const config = {
5555
warning.code?.startsWith("a11y") ||
5656
warning.message.includes("A11y") ||
5757
// md-generated doc content is full of `<video ... />` style self-closing tags
58-
warning.code === "element_invalid_self_closing_tag"
58+
warning.code === "element_invalid_self_closing_tag" ||
59+
// initial-value capture of props is intentional here: doc pages pass
60+
// static props (same semantics as the pre-runes svelte 4 code)
61+
warning.code === "state_referenced_locally"
5962
) {
6063
/// Too noisy
6164
return;

‎src/doc_builder/autodoc.py‎

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -194,29 +194,36 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l
194194
object_doc = remove_example_tags(object_doc)
195195
object_doc = hashlink_example_codeblock(object_doc, anchor)
196196

197-
svelte_str = "<docstring>"
198-
svelte_str += f"<name>{name}</name>"
199-
svelte_str += f"<anchor>{anchor}</anchor>"
197+
# Metadata props are emitted directly as svelte component attributes. Their values
198+
# are JS strings/JSON, where `{`, `<` and `#` are safe, so the MDX escaping
199+
# (see `convert_special_chars`) is undone; `\_` needs no markdown escaping either.
200+
def prop_value(text):
201+
return text.replace("&amp;lcub;", "{").replace("&amp;lt;", "<").replace("&amp;num;", "#")
202+
203+
name = prop_value(str(name)).replace("\\_", "_")
204+
svelte_str = f"<Docstring name={{{json.dumps(name)}}}"
205+
svelte_str += f" anchor={{{json.dumps(prop_value(str(anchor)))}}}"
200206
if source_link:
201-
svelte_str += f"<source>{source_link}</source>"
202-
svelte_str += f"<parameters>{json.dumps(signature)}</parameters>"
207+
svelte_str += f" source={{{json.dumps(prop_value(source_link))}}}"
208+
svelte_str += f" parameters={{{prop_value(json.dumps(signature))}}}"
203209
if is_getset_desc:
204-
svelte_str += "<isgetsetdescriptor>"
210+
svelte_str += " isGetSetDescriptor={true}"
211+
# the open tag must end with `>\n`: attribute values are single-line JSON, so the
212+
# kit preprocessor (kit/preprocessors/docstring.js) can find the end of the tag
213+
# unambiguously. The body carries the markdown-bearing sections, which the kit
214+
# preprocessor renders (mdsvex) into the remaining props.
215+
svelte_str += ">\n"
205216

206217
if parameters is not None:
207-
parameters_str = ""
208218
groups = _re_parameter_group.split(parameters)
209219
group_default = groups.pop(0)
210-
parameters_str += f"<paramsdesc>{group_default}</paramsdesc>"
211-
n_groups = len(groups) // 2
212-
for idx in range(n_groups):
213-
id = idx + 1
220+
svelte_str += f"<paramsdesc>{group_default}</paramsdesc>"
221+
for idx in range(len(groups) // 2):
214222
title, group = groups[2 * idx], groups[2 * idx + 1]
215-
parameters_str += f"<paramsdesc{id}title>{title}</paramsdesc{id}title>"
216-
parameters_str += f"<paramsdesc{id}>{group}</paramsdesc{id}>"
217-
218-
svelte_str += parameters_str
219-
svelte_str += f"<paramgroups>{n_groups}</paramgroups>"
223+
svelte_str += "<paramsgroup>"
224+
svelte_str += f"<paramsgrouptitle>{title}</paramsgrouptitle>"
225+
svelte_str += f"<paramsgroupdesc>{group}</paramsgroupdesc>"
226+
svelte_str += "</paramsgroup>"
220227

221228
if returntype is not None:
222229
svelte_str += f"<rettype>{returntype}</rettype>"
@@ -233,7 +240,7 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l
233240
if raisederrors is not None:
234241
svelte_str += f"<raisederrors>{raisederrors}</raisederrors>"
235242

236-
svelte_str += "</docstring>"
243+
svelte_str += "</Docstring>"
237244

238245
return svelte_str + f"\n{object_doc}\n"
239246

0 commit comments

Comments
 (0)