sphinx ext: include autosummary templates and extensions from cookiecutter-scverse (#60)

Author: ilia-kats

CHANGELOG.md

@@ -10,7 +10,11 @@ and this project adheres to [Semantic Versioning][].
 
 ## [Unreleased]
 
-## [0.1.0]
+### Added
+
+- The sphinx extension ships templates for `sphinx.ext.autosummary` that were previously part of `cookiecutter-scverse`.
+
+## [0.1.1]
 
 ### Fixed
 

docs/sphinx_ext.md

@@ -9,6 +9,13 @@ For example, using the [deprecated decorator](#scverse_misc.deprecated) adds a d
 To use the extension, add `"scverse_misc.sphinx_ext"` to your extensions array in `conf.py`.
 The extension requires `scverse_misc` to be installed with the `sphinx` extra.
 
+The extension also ships and enables several templates for `sphinx.ext.autosummary`.
+These were previously part of the [scverse cookiecutter template](https://github.com/scverse/cookiecutter-scverse).
+This keeps the cookiecutter template lean and ensures that scverse packages have a consistent documentation design, even if some packages are not updating the cookiecutter template.
+If you have custom autosummary templates in your package, they will still be used.
+
+Similarly, support for prose return sections in NumPy-style docstrings, previously part of cookiecutter-scverse, is now provided here.
+
 # Examples
 
 (example-deprecating-a-function)=

pyproject.toml

@@ -29,7 +29,7 @@ dependencies = [
 optional-dependencies.datasets = [ "anndata", "pooch", "pyyaml", "tqdm" ]
 optional-dependencies.settings = [ "pydantic-settings", "python-dotenv" ]
 optional-dependencies.spatialdata = [ "spatialdata" ]
-optional-dependencies.sphinx = [ "pydocstring-rs>=0.1.13", "sphinx>=9" ]
+optional-dependencies.sphinx = [ "jinja2", "pydocstring-rs>=0.1.13", "sphinx>=9" ]
 # https://docs.pypi.org/project_metadata/#project-urls
 urls.Documentation = "https://scverse-misc.readthedocs.io/"
 urls.Homepage = "https://github.com/scverse/scverse-misc"

src/scverse_misc/sphinx_ext/__init__.py

@@ -1,19 +1,38 @@
 from __future__ import annotations
 
+import re
 import sys
 import textwrap
 import warnings
 from importlib.metadata import version
+from pathlib import Path
 from textwrap import indent
 from types import MethodType
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Any, Generic, Literal, cast, get_origin
 
 if sys.version_info >= (3, 13):
     from warnings import deprecated as deprecated
 else:
     from typing_extensions import deprecated as deprecated
 
-from pydocstring import Docstring, Parameter, Return, Section, SectionKind, Style, emit_google, emit_numpy, parse
+from jinja2.defaults import DEFAULT_FILTERS  # type: ignore[attr-defined]
+from jinja2.utils import import_string
+from pydocstring import (
+    Docstring,
+    GoogleDocstring,
+    NumPyDocstring,
+    NumPySectionKind,
+    Parameter,
+    PlainDocstring,
+    Return,
+    Section,
+    SectionKind,
+    Style,
+    emit_google,
+    emit_numpy,
+    parse,
+)
+from sphinx.ext.napoleon import NumpyDocstring  # type: ignore[attr-defined]
 
 from .._deprecated import Deprecation, deprecated_arg
 from .._extensions import _NSInfo
@@ -30,6 +49,8 @@
 
 
 if TYPE_CHECKING:
+    from collections.abc import Generator, Iterable
+
     from sphinx.application import Sphinx
     from sphinx.ext.autodoc import Options as AutodocOptions
     from sphinx.ext.autodoc import _AutodocObjType  # type: ignore[attr-defined]
@@ -43,10 +64,58 @@ def setup(app: Sphinx) -> ExtensionMetadata:  # noqa: D103
     app.setup_extension("sphinx.ext.autodoc")
     # To go first, we use a lower number than napoleon (which uses the default, 500)
     app.connect("autodoc-process-docstring", _process_docstring, priority=100)
+    app.connect("autodoc-process-bases", _skip_private_bases)
+
+    DEFAULT_FILTERS["member_type"] = _member_type
+
+    app.config.templates_path = list(app.config.templates_path) + [str(Path(__file__).parent / "templates")]
+
+    NumpyDocstring._parse_returns_section = _parse_returns_section  # type: ignore[assignment]
 
     return {"version": version("scverse-misc"), "parallel_read_safe": True}
 
 
+def _process_return(lines: Iterable[str]) -> Generator[str, None, None]:
+    for line in lines:
+        if m := re.fullmatch(r"(?P<param>\w+)\s+:\s+(?P<type>[\w.]+)", line):
+            yield f"-{m['param']} (:class:`~{m['type']}`)"
+        else:
+            yield line
+
+
+def _parse_returns_section(self: NumpyDocstring, section: str) -> list[str]:
+    """Add support for prose return sections in Numpy-style docstrings."""
+    lines_raw = self._dedent(self._consume_to_next_section())
+    if lines_raw[0] == ":":
+        del lines_raw[0]
+    lines = self._format_block(":returns: ", list(_process_return(lines_raw)))
+    if lines and lines[-1]:
+        lines.append("")
+    return lines
+
+
+def _skip_private_bases(app: Sphinx, name: str, obj: type, _unused: Any, bases: list[type]) -> None:  # noqa: ANN401
+    bases[:] = [b for b in bases if b is not object and get_origin(b) is not Generic and not b.__name__.startswith("_")]
+
+
+def _member_type(obj_path: str) -> Literal["method", "property", "attribute"]:
+    """Determine object member type.
+
+    E.g.: `.. auto{{ fullname | member_type }}::`
+    """
+    # https://jinja.palletsprojects.com/en/stable/api/#custom-filters
+    cls_path, member_name = obj_path.rsplit(".", 1)
+    cls = import_string(cls_path)
+    member = getattr(cls, member_name, None)
+    match member:
+        case property():
+            return "property"
+        case _ if callable(member):
+            return "method"
+        case _:
+            return "attribute"
+
+
 def _process_docstring(
     app: Sphinx, objtype: _AutodocObjType, name: str, obj: object, options: AutodocOptions, lines: list[str]
 ) -> None:
@@ -62,13 +131,35 @@ def _process_docstring(
             if hasattr(obj, ATTR_DEPRECATED) and isinstance(msg := getattr(obj, ATTR_DEPRECATED, None), Deprecation):
                 _process_deprecated_function(app, msg, lines)
             if (args := getattr(obj, ATTR_DEPRECATED_ARG, None)) is not None:
-                _process_deprecated_args(args, lines)
+                _process_deprecated_args(app, args, lines)
         case "data" if isinstance(obj, Settings):
             _process_settings_object(obj, name, lines)
 
 
-def _emit_docstring(app: Sphinx, model: Docstring, lines: list[str]) -> None:
+def _emit_docstring(
+    app: Sphinx,
+    model: Docstring,
+    lines: list[str],
+    parsed: NumPyDocstring | GoogleDocstring | PlainDocstring | None = None,
+) -> None:
     """Emit a docstring compatible with the user settings (i.e. renderable with the chosen napoleon settings)."""
+    # pydocstring doesn't support prose return sections and tries to parse them as NumPy style.
+    # Copy the original return section verbatim.
+    if parsed is not None and isinstance(parsed, NumPyDocstring):
+        sections = model.sections
+        return_section = None
+        for i, section in enumerate(sections):
+            if section.kind == SectionKind.RETURNS:
+                return_section = i
+                break
+
+        if return_section is not None:
+            ast_section = parsed.sections[return_section]
+            assert ast_section.section_kind == NumPySectionKind.RETURNS
+            text = "\n".join(parsed.source[ast_section.range.start : ast_section.range.end].splitlines()[2:])
+            sections[i] = Section(SectionKind.UNKNOWN, unknown_name=ast_section.header_name.text, body=text)
+            model.sections = sections
+
     if getattr(app.config, "napoleon_google_docstring", True):
         doc = emit_google(model)
     elif getattr(app.config, "napoleon_numpy_docstring", True):
@@ -91,10 +182,10 @@ def _process_deprecated_function(app: Sphinx, msg: Deprecation, lines: list[str]
     if model.extended_summary is not None:
         notice += f"\n\n{model.extended_summary}"
     model.extended_summary = notice
-    _emit_docstring(app, model, lines)
+    _emit_docstring(app, model, lines, parsed)
 
 
-def _process_deprecated_args(deprecations: list[deprecated_arg], lines: list[str]) -> None:
+def _process_deprecated_args(app: Sphinx, deprecations: list[deprecated_arg], lines: list[str]) -> None:
     parsed = parse("\n".join(lines))
     if parsed.style is Style.PLAIN:
         return
@@ -124,15 +215,8 @@ def _process_deprecated_args(deprecations: list[deprecated_arg], lines: list[str
         sections = model.sections
         sections[s] = Section(section.kind, parameters=params)
         model.sections = sections
-    match parsed.style:
-        case Style.GOOGLE:
-            doc = emit_google(model)
-        case Style.NUMPY:
-            doc = emit_numpy(model)
-        case _:  # pragma: no cover
-            raise AssertionError
 
-    lines[:] = doc.strip("\n").splitlines()
+    _emit_docstring(app, model, lines, parsed)
 
 
 _settings_docstring_template = """Allows users to customize settings for the `{package}` package.

src/scverse_misc/sphinx_ext/templates/autosummary/class.rst

@@ -0,0 +1,54 @@
+{{ fullname | escape | underline }}
+
+.. currentmodule:: {{ module }}
+
+.. add toctree option to make autodoc generate the pages
+
+.. autoclass:: {{ fullname }}
+
+{% set methods = methods | select("ne", "__init__") | list %}
+
+{% block attributes %}
+{% for item in attributes %}
+{% if loop.first %}
+    Attributes table
+    ~~~~~~~~~~~~~~~~
+
+    .. autosummary::
+{% endif %}
+        ~{{ name }}.{{ item }}
+{%- endfor %}
+{% endblock %}
+
+{% block methods %}
+{% for item in methods %}
+{% if loop.first %}
+    Methods table
+    ~~~~~~~~~~~~~
+
+    .. autosummary::
+{% endif %}
+        ~{{ name }}.{{ item }}
+{% endfor %}
+{% endblock %}
+
+{% block attributes_documentation %}
+{% for item in attributes %}
+{% if loop.first %}
+    Attributes
+    ~~~~~~~~~~
+
+{% endif %}
+    .. auto{{ [fullname, item] | join(".") | member_type }}:: {{ [fullname, item] | join(".") }}
+{%- endfor %}
+{% endblock %}
+
+{% block methods_documentation %}
+{% for item in methods %}
+{% if loop.first %}
+    Methods
+    ~~~~~~~
+{% endif %}
+    .. automethod:: {{ [fullname, item] | join(".") }}
+{%- endfor %}
+{% endblock %}

src/scverse_misc/sphinx_ext/templates/autosummary/module.rst

@@ -0,0 +1,62 @@
+{{ fullname | escape | underline}}
+
+.. automodule:: {{ fullname }}
+
+    {% block attributes %}
+    {% if attributes %}
+    .. rubric:: {{ _('Module Attributes') }}
+
+    .. autosummary::
+    {% for item in attributes %}
+        {{ item }}
+    {%- endfor %}
+    {% endif %}
+    {% endblock %}
+
+    {% block functions %}
+    {% if functions %}
+    .. rubric:: {{ _('Functions') }}
+
+    .. autosummary::
+        :toctree:
+    {% for item in functions %}
+        {{ item }}
+    {%- endfor %}
+    {% endif %}
+    {% endblock %}
+
+    {% block classes %}
+    {% if classes %}
+    .. rubric:: {{ _('Classes') }}
+
+    .. autosummary::
+        :toctree:
+    {% for item in classes %}
+        {{ item }}
+    {%- endfor %}
+    {% endif %}
+    {% endblock %}
+
+    {% block exceptions %}
+    {% if exceptions %}
+    .. rubric:: {{ _('Exceptions') }}
+
+    .. autosummary::
+        :toctree:
+    {% for item in exceptions %}
+        {{ item }}
+    {%- endfor %}
+    {% endif %}
+    {% endblock %}
+
+{% block modules %}
+{% if modules %}
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+{% for item in modules %}
+   {{ item }}
+{%- endfor %}
+{% endif %}
+{% endblock %}

tests/deprecation_decorator/test_sphinx.py

@@ -40,6 +40,15 @@ def docstring(request: pytest.FixtureRequest, docstring_style: Literal["google",
                 baz
             keyword_only_default
                 foobar
+
+            Returns
+            -------
+            This is a prose returns section.
+
+            :attr:`~module.ClassName.attr1`
+                First attribute
+            :attr:`~module.ClassName.attr2`
+                Second attribute
             """
         case "long_googlestyle":
             if docstring_style == "numpy":
@@ -67,10 +76,19 @@ def test_deprecation_decorator(
     offset = 0 if docstring is None else 2
 
     if docstring is not None:
-        lines_orig = docstring.expandtabs().splitlines()
+        lines_orig = inspect.cleandoc(docstring).expandtabs().splitlines()
         assert lines[0] == lines_orig[0]
         assert len(lines[1].strip()) == 0, "expected empty line following summary"
 
+        try:
+            orig_returns_offset = lines_orig.index("Returns")
+        except ValueError:
+            pass
+        else:
+            returns_offset = lines.index("Returns")
+            for roffset in range(max(len(lines_orig) - orig_returns_offset, len(lines) - returns_offset)):
+                assert lines[returns_offset + roffset] == lines_orig[orig_returns_offset + roffset]
+
     assert lines[offset].startswith(".. version-deprecated")
     if msg is None:
         assert len(lines) == offset + 1 or not lines[offset + 1].startswith("   ")
@@ -85,7 +103,7 @@ def test_deprecation_decorator(
     ("positional_only_no_default", "positional_only_default", "positional_or_keyword_default", "keyword_only_default"),
 )
 def test_deprecated_arg_decorator(
-    parser: type[GoogleDocstring | NumpyDocstring], func: Callable[..., int], msg: str | None, arg: str
+    app: Sphinx, parser: type[GoogleDocstring | NumpyDocstring], func: Callable[..., int], msg: str | None, arg: str
 ) -> None:
     deprecated_func = deprecated_arg(arg, Deprecation("2.718", msg or ""))(func)
     with pytest.warns(FutureWarning, match=f"{arg} is deprecated"):
@@ -100,7 +118,7 @@ def test_deprecated_arg_decorator(
         return
 
     lines = (inspect.getdoc(deprecated_func) or "").splitlines()
-    sphinx_ext._process_deprecated_args(getattr(deprecated_func, ATTR_DEPRECATED_ARG), lines)
+    sphinx_ext._process_deprecated_args(app, getattr(deprecated_func, ATTR_DEPRECATED_ARG), lines)
     lines = parser(lines).lines()
 
     prefix = f":param {arg}:"