From e74731dee1efd1d4354b3aae62db1318676cbcf6 Mon Sep 17 00:00:00 2001 From: Mishig Date: Sat, 4 Jul 2026 16:02:31 +0200 Subject: [PATCH 1/2] refactor: emit the svelte component directly from python autodoc.py previously serialized docstring metadata into a zoo of custom tags (...N) that kit/preprocessors/ docstring.js regex-parsed back into a component. Python now emits the component open tag with all metadata props inline (name/anchor/source/parameters/isGetSetDescriptor as final JSON values, MDX escaping undone since JS strings need none); the component body carries only the markdown-bearing sections, which the kit preprocessor renders with mdsvex into the remaining props and closes the component. - parameter groups become nested blocks instead of numbered tags + a count - the rendered-
    parsing stays (documented): parameter lists must be rendered as one markdown list to keep exact tight/loose list semantics - fixes a long-standing tag mismatch: python emits but the old regex matched , silently dropping yield descriptions Verified byte-identical output (48/48 pages, modulo hashed asset names and nondeterministic python object addresses) on the accelerate e2e build; python test suite updated and passing. Co-Authored-By: Claude Fable 5 --- kit/preprocessors/docstring.js | 234 +++++++++++++++------------------ src/doc_builder/autodoc.py | 41 +++--- tests/test_autodoc.py | 16 ++- 3 files changed, 138 insertions(+), 153 deletions(-) diff --git a/kit/preprocessors/docstring.js b/kit/preprocessors/docstring.js index fe61569c..5d03081b 100644 --- a/kit/preprocessors/docstring.js +++ b/kit/preprocessors/docstring.js @@ -3,46 +3,50 @@ import htmlparser2 from "htmlparser2"; import { replaceAsync, renderSvelteChars, generateTagRegex } from "./utils.js"; import { mdsvexPreprocess } from "./mdsvex/index.js"; -// Preprocessor that converts markdown into Docstring -// svelte component using mdsvexPreprocess +const REGEX_PARAMSDESC = generateTagRegex("paramsdesc"); +const REGEX_PARAMSGROUP = generateTagRegex("paramsgroup", true); +const REGEX_PARAMSGROUP_TITLE = generateTagRegex("paramsgrouptitle"); +const REGEX_PARAMSGROUP_DESC = generateTagRegex("paramsgroupdesc"); +const REGEX_RETDESC = generateTagRegex("retdesc"); +const REGEX_RETTYPE = generateTagRegex("rettype"); +// note: fixes a long-standing tag mismatch (python emits `yielddesc`, the old regex +// matched `yieldesc`), which silently dropped yield descriptions +const REGEX_YIELDESC = generateTagRegex("yielddesc"); +const REGEX_YIELDTYPE = generateTagRegex("yieldtype"); +const REGEX_RAISEDESC = generateTagRegex("raises"); +const REGEX_RAISETYPE = generateTagRegex("raisederrors"); +const REGEX_TIP = /(((?!).)*)<\/Tip>/gms; +const REGEX_CHANGED = + /<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms; + +/** + * Python (doc_builder/autodoc.py) emits the `` component + * directly; its body carries the markdown-bearing sections (parameter descriptions, + * return/yield/raise types & descriptions), which must be rendered with the same mdsvex + * pipeline as the rest of the page. This preprocessor renders those sections into the + * remaining component props and closes the component. + */ export const docstringPreprocess = { markup: async ({ content, filename }) => { - const REGEX_DOCSTRING = generateTagRegex("docstring", true); - const REGEX_NAME = generateTagRegex("name"); - const REGEX_ANCHOR = generateTagRegex("anchor"); - const REGEX_SIGNATURE = generateTagRegex("parameters"); - const REGEX_PARAMSDESC = generateTagRegex("paramsdesc"); - const REGEX_PARAMSGROUPS = generateTagRegex("paramgroups"); - const REGEX_RETDESC = generateTagRegex("retdesc"); - const REGEX_RETTYPE = generateTagRegex("rettype"); - const REGEX_YIELDESC = generateTagRegex("yieldesc"); - const REGEX_YIELDTYPE = generateTagRegex("yieldtype"); - const REGEX_RAISEDESC = generateTagRegex("raises"); - const REGEX_RAISETYPE = generateTagRegex("raisederrors"); - const REGEX_SOURCE = generateTagRegex("source"); - const REGEX_TIP = /(((?!).)*)<\/Tip>/gms; - const REGEX_CHANGED = - /<(Added|Changed|Deprecated) version="([0-9.v]+)" ?\/?>((((?!<(Added|Changed|Deprecated) version="([0-9.v]+)"\/?>).)*)<\/(Added|Changed|Deprecated)>)?/gms; - const REGEX_IS_GETSET_DESC = //ms; - - content = await replaceAsync(content, REGEX_DOCSTRING, async (_, docstringBody) => { - docstringBody = renderSvelteChars(docstringBody); - - const name = docstringBody.match(REGEX_NAME)[1]; - const anchor = docstringBody.match(REGEX_ANCHOR)[1]; - const signature = docstringBody.match(REGEX_SIGNATURE)[1]; - - let svelteComponent = `` followed by a newline: attribute values are + // single-line JSON, which may contain `>` inside strings but never a raw newline. + const REGEX_DOCSTRING = /\n)[\s\S])*)>\n([\s\S]*?)<\/Docstring>/g; + + content = await replaceAsync(content, REGEX_DOCSTRING, async (_, metadataAttrs, body) => { + body = renderSvelteChars(body); + + // parameter anchors are prefixed with the object's anchor + const anchor = metadataAttrs.match(/ anchor=\{"((?:[^"\\]|\\.)*)"\}/)?.[1] ?? ""; + + let svelteComponent = ` components that are inside parameter descriptions - code = code.replace(REGEX_TIP, (_, isWarning, tipContent) => { + code = code.replace(REGEX_TIP, (_tip, isWarning, tipContent) => { const color = isWarning ? "orange" : "green"; return `
    { + const color = /Added|Changed/.test(componentType) ? "green" : "orange"; + if (!descriptionContent) { + descriptionContent = ""; + } + return `
    ${componentType} in ${version}

    ${descriptionContent}
    `; - }); - - const dom = htmlparser2.parseDocument(code); - const lists = domUtils.getElementsByTagName("ul", dom); - if (lists.length) { - const list = lists[0]; - const result = []; - for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) { - const nameEl = domUtils.getElementsByTagName("strong", childEl)[0]; - const name = domUtils.innerText(nameEl); - const paramAnchor = `${anchor}.${name}`; - let description = domUtils.getInnerHTML(childEl).trim(); - - // strip enclosing paragraph tags

    &

    - if (description.startsWith("

    ")) { - description = description.slice("

    ".length); - } - if (description.endsWith("

    ")) { - description = description.slice(0, -"

    ".length); - } - - result.push({ anchor: paramAnchor, description, name }); } + ); + + const result = parseRenderedParamsList(code, anchor); + if (result) { svelteComponent += ` parametersDescription={${JSON.stringify(result)}} `; } } - if (docstringBody.match(REGEX_SOURCE)) { - const source = docstringBody.match(REGEX_SOURCE)[1]; - svelteComponent += ` source={${JSON.stringify(source)}} `; - } - - if (docstringBody.match(REGEX_RETDESC)) { - const retDesc = docstringBody.match(REGEX_RETDESC)[1]; + if (body.match(REGEX_RETDESC)) { + const retDesc = body.match(REGEX_RETDESC)[1]; const { code } = await mdsvexPreprocess.markup({ content: retDesc, filename }); svelteComponent += ` returnDescription={${JSON.stringify(code)}} `; } - if (docstringBody.match(REGEX_RETTYPE)) { - const retType = docstringBody.match(REGEX_RETTYPE)[1]; + if (body.match(REGEX_RETTYPE)) { + const retType = body.match(REGEX_RETTYPE)[1]; const { code } = await mdsvexPreprocess.markup({ content: retType, filename }); svelteComponent += ` returnType={${JSON.stringify(code)}} `; } - if (docstringBody.match(REGEX_YIELDESC)) { - const yieldDesc = docstringBody.match(REGEX_YIELDESC)[1]; + if (body.match(REGEX_YIELDESC)) { + const yieldDesc = body.match(REGEX_YIELDESC)[1]; const { code } = await mdsvexPreprocess.markup({ content: yieldDesc, filename }); svelteComponent += ` returnDescription={${JSON.stringify(code)}} `; } - if (docstringBody.match(REGEX_YIELDTYPE)) { - const yieldType = docstringBody.match(REGEX_YIELDTYPE)[1]; + if (body.match(REGEX_YIELDTYPE)) { + const yieldType = body.match(REGEX_YIELDTYPE)[1]; const { code } = await mdsvexPreprocess.markup({ content: yieldType, filename }); svelteComponent += ` returnType={${JSON.stringify(code)}} isYield={true} `; } - if (docstringBody.match(REGEX_RAISEDESC)) { - const raiseDesc = docstringBody.match(REGEX_RAISEDESC)[1]; + if (body.match(REGEX_RAISEDESC)) { + const raiseDesc = body.match(REGEX_RAISEDESC)[1]; const { code } = await mdsvexPreprocess.markup({ content: raiseDesc, filename }); svelteComponent += ` raiseDescription={${JSON.stringify(code)}} `; } - if (docstringBody.match(REGEX_RAISETYPE)) { - const raiseType = docstringBody.match(REGEX_RAISETYPE)[1]; + if (body.match(REGEX_RAISETYPE)) { + const raiseType = body.match(REGEX_RAISETYPE)[1]; const { code } = await mdsvexPreprocess.markup({ content: raiseType, filename }); svelteComponent += ` raiseType={${JSON.stringify(code)}} `; } - if (docstringBody.match(REGEX_IS_GETSET_DESC)) { - svelteComponent += ` isGetSetDescriptor={true} `; + const parameterGroups = []; + for (const [, groupBody] of body.matchAll(REGEX_PARAMSGROUP)) { + const title = groupBody.match(REGEX_PARAMSGROUP_TITLE)[1]; + const groupContent = groupBody.match(REGEX_PARAMSGROUP_DESC)[1]; + const { code } = await mdsvexPreprocess.markup({ content: groupContent, filename }); + parameterGroups.push({ + title, + parametersDescription: parseRenderedParamsList(code, anchor) ?? [], + }); } - - if (docstringBody.match(REGEX_PARAMSGROUPS)) { - const nParamGroups = parseInt(docstringBody.match(REGEX_PARAMSGROUPS)[1]); - if (nParamGroups > 0) { - const parameterGroups = []; - for (let groupId = 1; groupId <= nParamGroups; groupId++) { - const REGEX_GROUP_TITLE = new RegExp( - `(((?!).)*)`, - "ms" - ); - const REGEX_GROUP_CONTENT = new RegExp( - `(((?!).)*)`, - "ms" - ); - const title = docstringBody.match(REGEX_GROUP_TITLE)[1]; - const content = docstringBody.match(REGEX_GROUP_CONTENT)[1]; - const { code } = await mdsvexPreprocess.markup({ content, filename }); - const dom = htmlparser2.parseDocument(code); - const lists = domUtils.getElementsByTagName("ul", dom); - const result = []; - if (lists.length) { - const list = lists[0]; - for (const childEl of list.childNodes.filter(({ type }) => type === "tag")) { - const nameEl = domUtils.getElementsByTagName("strong", childEl)[0]; - const name = domUtils.innerText(nameEl); - const paramAnchor = `${anchor}.${name}`; - let description = domUtils.getInnerHTML(childEl).trim(); - - // strip enclosing paragraph tags

    &

    - if (description.startsWith("

    ")) { - description = description.slice("

    ".length); - } - if (description.endsWith("

    ")) { - description = description.slice(0, -"

    ".length); - } - - result.push({ anchor: paramAnchor, description, name }); - } - } - parameterGroups.push({ title, parametersDescription: result }); - } - svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `; - } + if (parameterGroups.length) { + svelteComponent += ` parameterGroups={${JSON.stringify(parameterGroups)}} `; } svelteComponent += ` />\n`; @@ -191,8 +140,35 @@ export const docstringPreprocess = { }; /** - * The mdx file contains unnecessarily escaped underscores in the docstring's name + * The parameter descriptions are authored as a markdown bullet list (`- **name** -- desc`) + * and must be rendered as one list to keep exact markdown semantics (tight/loose lists); + * this parses the rendered `
      ` back into per-parameter structured data for the + * `Docstring` component (tooltips, per-parameter anchors). + * @param {string} code rendered html + * @param {string} anchor the object's anchor, used as prefix for parameter anchors */ -function unescapeUnderscores(content) { - return content.replace(/\\_/g, "_"); +function parseRenderedParamsList(code, anchor) { + const dom = htmlparser2.parseDocument(code); + const lists = domUtils.getElementsByTagName("ul", dom); + if (!lists.length) { + return undefined; + } + const result = []; + for (const childEl of lists[0].childNodes.filter(({ type }) => type === "tag")) { + const nameEl = domUtils.getElementsByTagName("strong", childEl)[0]; + const name = domUtils.innerText(nameEl); + const paramAnchor = `${anchor}.${name}`; + let description = domUtils.getInnerHTML(childEl).trim(); + + // strip enclosing paragraph tags

      &

      + if (description.startsWith("

      ")) { + description = description.slice("

      ".length); + } + if (description.endsWith("

      ")) { + description = description.slice(0, -"

      ".length); + } + + result.push({ anchor: paramAnchor, description, name }); + } + return result; } diff --git a/src/doc_builder/autodoc.py b/src/doc_builder/autodoc.py index 9bc4610b..083cc85d 100644 --- a/src/doc_builder/autodoc.py +++ b/src/doc_builder/autodoc.py @@ -194,29 +194,36 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l object_doc = remove_example_tags(object_doc) object_doc = hashlink_example_codeblock(object_doc, anchor) - svelte_str = "" - svelte_str += f"{name}" - svelte_str += f"{anchor}" + # Metadata props are emitted directly as svelte component attributes. Their values + # are JS strings/JSON, where `{`, `<` and `#` are safe, so the MDX escaping + # (see `convert_special_chars`) is undone; `\_` needs no markdown escaping either. + def prop_value(text): + return text.replace("&lcub;", "{").replace("&lt;", "<").replace("&num;", "#") + + name = prop_value(str(name)).replace("\\_", "_") + svelte_str = f"{source_link}" - svelte_str += f"{json.dumps(signature)}" + svelte_str += f" source={{{json.dumps(prop_value(source_link))}}}" + svelte_str += f" parameters={{{prop_value(json.dumps(signature))}}}" if is_getset_desc: - svelte_str += "" + svelte_str += " isGetSetDescriptor={true}" + # the open tag must end with `>\n`: attribute values are single-line JSON, so the + # kit preprocessor (kit/preprocessors/docstring.js) can find the end of the tag + # unambiguously. The body carries the markdown-bearing sections, which the kit + # preprocessor renders (mdsvex) into the remaining props. + svelte_str += ">\n" if parameters is not None: - parameters_str = "" groups = _re_parameter_group.split(parameters) group_default = groups.pop(0) - parameters_str += f"{group_default}" - n_groups = len(groups) // 2 - for idx in range(n_groups): - id = idx + 1 + svelte_str += f"{group_default}" + for idx in range(len(groups) // 2): title, group = groups[2 * idx], groups[2 * idx + 1] - parameters_str += f"{title}" - parameters_str += f"{group}" - - svelte_str += parameters_str - svelte_str += f"{n_groups}" + svelte_str += "" + svelte_str += f"{title}" + svelte_str += f"{group}" + svelte_str += "" if returntype is not None: svelte_str += f"{returntype}" @@ -233,7 +240,7 @@ def get_signature_component_svelte(name, anchor, signature, object_doc, source_l if raisederrors is not None: svelte_str += f"{raisederrors}" - svelte_str += "" + svelte_str += "" return svelte_str + f"\n{object_doc}\n" diff --git a/tests/test_autodoc.py b/tests/test_autodoc.py index 329e8b7c..c82057d3 100644 --- a/tests/test_autodoc.py +++ b/tests/test_autodoc.py @@ -220,7 +220,7 @@ def test_get_signature_component(self): ] object_doc = TEST_DOCSTRING source_link = "test_link" - expected_signature_component = 'class transformers.BertweetTokenizertransformers.BertweetTokenizertest_link[{"name": "vocab_file", "val": ""}, {"name": "normalization", "val": " = False"}, {"name": "bos_token", "val": " = \'&lt;s>\'"}]- **vocab_file** (`str`) --\n Path to the vocabulary file.\n- **merges_file** (`str`) --\n Path to the merges file.\n- **normalization** (`bool`, _optional_, defaults to `False`) --\n Whether or not to apply a normalization preprocess.\n\n\n\nWhen building a sequence using special tokens, this is not the token that is used for the beginning of\nsequence. The token used is the `cls_token`.\n\n0`List[int]`List of [input IDs](../glossary.html#input-ids) with the appropriate special tokens.- ``ValuError`` -- this value error will be raised on wrong input type.``ValuError``\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n\n\n\n\n\n\n\n' + expected_signature_component = '\'"}]}>\n- **vocab_file** (`str`) --\n Path to the vocabulary file.\n- **merges_file** (`str`) --\n Path to the merges file.\n- **normalization** (`bool`, _optional_, defaults to `False`) --\n Whether or not to apply a normalization preprocess.\n\n\n\nWhen building a sequence using special tokens, this is not the token that is used for the beginning of\nsequence. The token used is the `cls_token`.\n\n`List[int]`List of [input IDs](../glossary.html#input-ids) with the appropriate special tokens.- ``ValuError`` -- this value error will be raised on wrong input type.``ValuError``\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n\n\n\n\n\n\n\n' self.assertEqual( get_signature_component_svelte(name, anchor, signature, object_doc, source_link), expected_signature_component, @@ -238,7 +238,7 @@ def test_get_signature_component(self): This tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. """ - expected_signature_component = 'class transformers.BertweetTokenizertransformers.BertweetTokenizertest_link[{"name": "vocab_file", "val": ""}, {"name": "normalization", "val": " = False"}, {"name": "bos_token", "val": " = \'&lt;s>\'"}]\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n' + expected_signature_component = '\'"}]}>\n\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n' self.assertEqual( get_signature_component_svelte(name, anchor, signature, object_doc_without_params_and_return, source_link), expected_signature_component, @@ -254,7 +254,7 @@ def test_get_signature_component(self): ] object_doc = TEST_DOCSTRING_WITH_PARAM_GROUPS source_link = "test_link" - expected_signature_component = 'class transformers.cool_functiontransformers.cool_functiontest_link[{"name": "param_a", "val": ""}, {"name": "param_b", "val": ""}, {"name": "cool_param_a", "val": ""}, {"name": "cool_param_b", "val": ""}]- **param_a** (`str`) --\n First default parameter\n- **param_b** (`int`) --\n Second default parameter\n\nNew group with cool parameters!\n\n- **cool_param_a** (`str`) --\n First cool parameter\n- **cool_param_b** (`int`) --\n Second cool parameter1\n\nBuilds something very cool!\n\n\n\n' + expected_signature_component = '\n- **param_a** (`str`) --\n First default parameter\n- **param_b** (`int`) --\n Second default parameter\n\nNew group with cool parameters!\n\n- **cool_param_a** (`str`) --\n First cool parameter\n- **cool_param_b** (`int`) --\n Second cool parameter\n\nBuilds something very cool!\n\n\n\n' self.assertEqual( get_signature_component_svelte(name, anchor, signature, object_doc, source_link), expected_signature_component, @@ -275,9 +275,10 @@ def test_document_object(self): page_info = {"package_name": "transformers"} model_output_doc = """ -class transformers.utils.ModelOutputtransformers.utils.ModelOutput""" - model_output_doc += f"{self.test_source_link}" - model_output_doc += """[{"name": "*args", "val": ""}, {"name": "**kwargs", "val": ""}] + + Base class for all model outputs as dataclass. Has a `__getitem__` that allows indexing by integer or slice (like a tuple) or strings (like a dictionary) that will ignore the `None` attributes. Otherwise behaves like a regular @@ -542,7 +543,8 @@ def test_autodoc_getset_descriptor(self): expected_documentation = """
      -contentNone[] + + Get the content of this `AddedToken`
      \n""" From 9e281bd4cb64ffdcb813d9d7a350b7b7a5e949b2 Mon Sep 17 00:00:00 2001 From: Mishig Date: Sat, 4 Jul 2026 16:02:31 +0200 Subject: [PATCH 2/2] fix(kit): silence state_referenced_locally warnings (runes follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This hunk was part of the runes migration verification but was left out of #795 (staged with `git add kit/src`, which misses kit/svelte.config.js). The flagged initial-value captures have the same semantics as the pre-runes svelte 4 code — doc pages pass static props — and the warning floods every downstream doc build log otherwise. Co-Authored-By: Claude Fable 5 --- kit/svelte.config.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kit/svelte.config.js b/kit/svelte.config.js index 5ae59234..37bdbfeb 100644 --- a/kit/svelte.config.js +++ b/kit/svelte.config.js @@ -55,7 +55,10 @@ const config = { warning.code?.startsWith("a11y") || warning.message.includes("A11y") || // md-generated doc content is full of `