👌 Enhance [](#id) references (#707)

This PR moves myst-parser in the direction of jupyter-book/myst-enhancement-proposals#10, in a relatively back compatible manner, and in a way that supports both docutils and sphinx.

It expands the capability of `[](#id)` to more than just linking to heading slugs, with the order of specificity being:

1. If it matches a local (to that document) "explicit" `std:ref` target, then link to that and stop
   - Note, currently only `std:ref` domain/type are supported, e.g. not `math` etc. That's more difficult and can come later
2. If it matches a local (to that document) "implicit" heading slug, then link to that and stop
3. If using docutils (i.e. single-page) build, then stop here and emit a `myst.xref_missing` warning
4. If using sphinx then create a `pending_xref` node, and hand-off to sphinx's "any" resolver, which takes effect once all documents have been read:
    - This first tries to resolve against a local (to the project) reference and, if matching, stops
    - Otherwise also try to match against any intersphinx reference
    - Otherwise emit a ~~`myst.ref`~~ `myst.xref_missing`  warning
 
If the text is explicit, e.g. `[text](#id)`, that text is used, otherwise a determination of implicit text is attempted, e.g. based on the section title or figure caption.

Author: chrisjsewell

codecov.yml

@@ -2,7 +2,7 @@ coverage:
   status:
     project:
       default:
-        target: 90%
+        target: 89%
         threshold: 0.5%
     patch:
       default:

myst_parser/mdit_to_docutils/base.py

@@ -248,36 +248,14 @@ def _render_initialise(self) -> None:
     def _render_finalise(self) -> None:
         """Finalise the render of the document."""
 
-        # attempt to replace id_link references with internal links
-        for refnode in findall(self.document)(nodes.reference):
-            if not refnode.get("id_link"):
-                continue
-            target = refnode["refuri"][1:]
-            if target in self._slug_to_section:
-                section_node = self._slug_to_section[target]
-                refnode["refid"] = section_node["ids"][0]
-
-                if not refnode.children:
-                    implicit_text = clean_astext(section_node[0])
-                    refnode += nodes.inline(
-                        implicit_text, implicit_text, classes=["std", "std-ref"]
-                    )
-            else:
-                self.create_warning(
-                    f"local id not found: {refnode['refuri']!r}",
-                    MystWarnings.XREF_MISSING,
-                    line=refnode.line,
-                    append_to=refnode,
-                )
-                refnode["refid"] = target
-            del refnode["refuri"]
-
-        if self._slug_to_section and self.sphinx_env:
-            # save for later reference resolution
-            self.sphinx_env.metadata[self.sphinx_env.docname]["myst_slugs"] = {
-                slug: (snode["ids"][0], clean_astext(snode[0]))
-                for slug, snode in self._slug_to_section.items()
-            }
+        # save for later reference resolution
+        slugs = {
+            slug: (snode.line, snode["ids"][0], clean_astext(snode[0]))
+            for slug, snode in self._slug_to_section.items()
+        }
+        self.document.myst_slugs = slugs
+        if slugs and self.sphinx_env:
+            self.sphinx_env.metadata[self.sphinx_env.docname]["myst_slugs"] = slugs
 
         # log warnings for duplicate reference definitions
         # "duplicate_refs": [{"href": "ijk", "label": "B", "map": [4, 5], "title": ""}],
@@ -795,7 +773,7 @@ def render_heading(self, token: SyntaxTreeNode) -> None:
             token.attrs.get("toc", None) == "false"
             or self.md_env.get("match_titles", None) is False
         ):
-            if self.md_env.get("match_titles", None) is False:
+            if token.attrs.get("toc", None) != "false":
                 # this can occur if a nested parse is performed by a directive
                 # (such as an admonition) which contains a header.
                 # this would break the document structure
@@ -984,8 +962,10 @@ def render_id_link(self, token: SyntaxTreeNode) -> None:
         self.copy_attributes(
             token, ref_node, ("class", "id", "reftitle"), aliases={"title": "reftitle"}
         )
-        with self.current_node_context(ref_node, append=True):
-            self.render_children(token)
+        self.current_node.append(ref_node)
+        if token.info != "auto" and not is_ellipsis(token):
+            with self.current_node_context(ref_node):
+                self.render_children(token)
 
     def render_internal_link(self, token: SyntaxTreeNode) -> None:
         """Render link token `[text](link "title")`,
@@ -1024,7 +1004,9 @@ def render_inventory_link(self, token: SyntaxTreeNode) -> None:
         href = self.md.normalizeLinkText(cast(str, token.attrGet("href") or ""))
 
         # note if the link had explicit text or not (autolinks are always implicit)
-        explicit = False if token.info == "auto" else bool(token.children)
+        explicit = (
+            (token.info != "auto") and bool(token.children) and not is_ellipsis(token)
+        )
 
         # split the href up into parts
         uri_parts = urlparse(href)
@@ -1897,3 +1879,12 @@ def compute_unique_slug(
         slug = f"{slug}-{i}"
         i += 1
     return slug
+
+
+def is_ellipsis(token: SyntaxTreeNode) -> bool:
+    """Check if a token content only contains an ellipsis."""
+    return (
+        len(token.children) == 1
+        and token.children[0].type == "text"
+        and token.children[0].content in ("...", "…")
+    )

myst_parser/mdit_to_docutils/sphinx_.py

@@ -16,7 +16,7 @@
 from sphinx.util import logging
 
 from myst_parser import inventory
-from myst_parser.mdit_to_docutils.base import DocutilsRenderer
+from myst_parser.mdit_to_docutils.base import DocutilsRenderer, is_ellipsis
 
 LOGGER = logging.getLogger(__name__)
 
@@ -46,25 +46,33 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None:
             destination = os.path.relpath(
                 os.path.join(include_dir, os.path.normpath(destination)), source_dir
             )
+        explicit = len(token.children or []) > 0 and not is_ellipsis(token)
         kwargs = {
             "refdoc": self.sphinx_env.docname,
             "reftype": "myst",
-            "refexplicit": len(token.children or []) > 0,
+            "refexplicit": explicit,
         }
         path_dest, *_path_ids = destination.split("#", maxsplit=1)
         path_id = _path_ids[0] if _path_ids else None
-        potential_path = (
-            Path(self.sphinx_env.doc2path(self.sphinx_env.docname)).parent / path_dest
-            if self.sphinx_env.srcdir  # not set in some test situations
-            else None
-        )
-        if path_dest == "./":
+
+        potential_path: None | Path
+        if path_dest.startswith("/"):
+            # here we are referencing a file relative to the source directory
+            potential_path = Path(self.sphinx_env.srcdir) / path_dest[1:]
+        elif path_dest == "./":
             # this is a special case, where we want to reference the current document
             potential_path = (
                 Path(self.sphinx_env.doc2path(self.sphinx_env.docname))
                 if self.sphinx_env.srcdir
                 else None
             )
+        else:
+            potential_path = (
+                Path(self.sphinx_env.doc2path(self.sphinx_env.docname)).parent
+                / path_dest
+                if self.sphinx_env.srcdir  # not set in some test situations
+                else None
+            )
         if potential_path and potential_path.is_file():
             docname = self.sphinx_env.path2doc(str(potential_path))
             if docname:
@@ -92,8 +100,9 @@ def render_internal_link(self, token: SyntaxTreeNode) -> None:
 
         inner_node = nodes.inline("", text, classes=classes)
         wrap_node.append(inner_node)
-        with self.current_node_context(inner_node):
-            self.render_children(token)
+        if explicit:
+            with self.current_node_context(inner_node):
+                self.render_children(token)
 
     def get_inventory_matches(
         self,

myst_parser/mdit_to_docutils/transforms.py

@@ -0,0 +1,145 @@
+"""Directives that can be applied to both Sphinx and docutils."""
+from __future__ import annotations
+
+import typing as t
+
+from docutils import nodes
+from docutils.transforms import Transform
+
+from myst_parser._compat import findall
+from myst_parser.mdit_to_docutils.base import clean_astext
+from myst_parser.warnings_ import MystWarnings, create_warning
+
+
+class ResolveAnchorIds(Transform):
+    """Directive for resolving `[name](#id)` type links."""
+
+    default_priority = 879  # this is the same as Sphinx's StandardDomain.process_doc
+
+    def apply(self, **kwargs: t.Any) -> None:
+        """Apply the transform."""
+        # gather the implicit heading slugs
+        # name -> (line, slug, title)
+        slugs: dict[str, tuple[int, str, str]] = getattr(
+            self.document, "myst_slugs", {}
+        )
+
+        # gather explicit references
+        # this follows the same logic as Sphinx's StandardDomain.process_doc
+        explicit: dict[str, tuple[str, None | str]] = {}
+        for name, is_explicit in self.document.nametypes.items():
+            if not is_explicit:
+                continue
+            labelid = self.document.nameids[name]
+            if labelid is None:
+                continue
+            if labelid is None:
+                continue
+            node = self.document.ids[labelid]
+            if isinstance(node, nodes.target) and "refid" in node:
+                # indirect hyperlink targets
+                node = self.document.ids.get(node["refid"])
+                labelid = node["names"][0]
+            if (
+                node.tagname == "footnote"
+                or "refuri" in node
+                or node.tagname.startswith("desc_")
+            ):
+                # ignore footnote labels, labels automatically generated from a
+                # link and object descriptions
+                continue
+
+            implicit_title = None
+            if node.tagname == "rubric":
+                implicit_title = clean_astext(node)
+            if implicit_title is None:
+                # handle sections and and other captioned elements
+                for subnode in node:
+                    if isinstance(subnode, (nodes.caption, nodes.title)):
+                        implicit_title = clean_astext(subnode)
+                        break
+            if implicit_title is None:
+                # handle definition lists and field lists
+                if (
+                    isinstance(node, (nodes.definition_list, nodes.field_list))
+                    and node.children
+                ):
+                    node = node[0]
+                if (
+                    isinstance(node, (nodes.field, nodes.definition_list_item))
+                    and node.children
+                ):
+                    node = node[0]
+                if isinstance(node, (nodes.term, nodes.field_name)):
+                    implicit_title = clean_astext(node)
+
+            explicit[name] = (labelid, implicit_title)
+
+        for refnode in findall(self.document)(nodes.reference):
+            if not refnode.get("id_link"):
+                continue
+
+            target = refnode["refuri"][1:]
+            del refnode["refuri"]
+
+            # search explicit first
+            if target in explicit:
+                ref_id, implicit_title = explicit[target]
+                refnode["refid"] = ref_id
+                if not refnode.children and implicit_title:
+                    refnode += nodes.inline(
+                        implicit_title, implicit_title, classes=["std", "std-ref"]
+                    )
+                elif not refnode.children:
+                    refnode += nodes.inline(
+                        "#" + target, "#" + target, classes=["std", "std-ref"]
+                    )
+                continue
+
+            # now search implicit
+            if target in slugs:
+                _, sect_id, implicit_title = slugs[target]
+                refnode["refid"] = sect_id
+                if not refnode.children and implicit_title:
+                    refnode += nodes.inline(
+                        implicit_title, implicit_title, classes=["std", "std-ref"]
+                    )
+                continue
+
+            # if still not found, and using sphinx, then create a pending_xref
+            if hasattr(self.document.settings, "env"):
+                from sphinx import addnodes
+
+                pending = addnodes.pending_xref(
+                    refdoc=self.document.settings.env.docname,
+                    refdomain=None,
+                    reftype="myst",
+                    reftarget=target,
+                    refwarn=True,
+                    refexplicit=bool(refnode.children),
+                )
+                inner_node = nodes.inline(
+                    "", "", classes=["xref", "myst"] + refnode["classes"]
+                )
+                for attr in ("ids", "names", "dupnames"):
+                    inner_node[attr] = refnode[attr]
+                inner_node += refnode.children
+                pending += inner_node
+                refnode.parent.replace(refnode, pending)
+                continue
+
+            # if still not found, and using docutils, then create a warning
+            # and simply output as a url
+
+            create_warning(
+                self.document,
+                f"'myst' reference target not found: {target!r}",
+                MystWarnings.XREF_MISSING,
+                line=refnode.line,
+                append_to=refnode,
+            )
+            refnode["refid"] = target
+            if not refnode.children:
+                refnode += nodes.inline(
+                    "#" + target, "#" + target, classes=["std", "std-ref"]
+                )

myst_parser/parsers/docutils_.py

@@ -26,6 +26,7 @@
     read_topmatter,
 )
 from myst_parser.mdit_to_docutils.base import DocutilsRenderer
+from myst_parser.mdit_to_docutils.transforms import ResolveAnchorIds
 from myst_parser.parsers.mdit import create_md_parser
 from myst_parser.warnings_ import MystWarnings, create_warning
 
@@ -247,6 +248,9 @@ class Parser(RstParser):
     config_section_dependencies = ("parsers",)
     translate_section_name = None
 
+    def get_transforms(self):
+        return super().get_transforms() + [ResolveAnchorIds]
+
     def parse(self, inputstring: str, document: nodes.document) -> None:
         """Parse source text.
 

myst_parser/parsers/sphinx_.py

@@ -13,6 +13,7 @@
     read_topmatter,
 )
 from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
+from myst_parser.mdit_to_docutils.transforms import ResolveAnchorIds
 from myst_parser.parsers.mdit import create_md_parser
 from myst_parser.warnings_ import create_warning
 
@@ -43,6 +44,9 @@ class MystParser(SphinxParser):
     config_section_dependencies = ("parsers",)
     translate_section_name = None
 
+    def get_transforms(self):
+        return super().get_transforms() + [ResolveAnchorIds]
+
     def parse(self, inputstring: str, document: nodes.document) -> None:
         """Parse source text.