Release: Fix changelog formatting (#5529)

# Improve release notes formatting / changelog conversion from rst to md

During our last release, we discovered issues with changelog formatting.
This PR improves and fixes several aspects:

## Changes
- Rewrite the changelog conversion logic to be more robust and
maintainable
- Fix indentation issues with nested bullet points
- Improve handling of long section headers
- Order bullet points alphabetically within sections for better
readability
- Use Sphinx `objects.inv` to resolve references and include links to
the documentation in _Markdown_
- Add tests to prevent formatting regressions
- Add pandoc as a dependency for Ubuntu CI builds
- Ensure documentation is built before generating changelog

Author: snejus

.github/workflows/ci.yaml

@@ -29,12 +29,13 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: poetry
 
-      - name: Install PyGobject dependencies on Ubuntu
+      - name: Install PyGobject and release script dependencies on Ubuntu
         if: matrix.platform == 'ubuntu-latest'
         run: |
           sudo apt update
-          sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev
-          poetry install --extras=replaygain --extras=reflink
+          sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev pandoc
+          poetry install --with=release --extras=docs --extras=replaygain --extras=reflink
+          poe docs
 
       - name: Install Python dependencies
         run: poetry install --only=main,test --extras=autobpm

.github/workflows/make_release.yaml

@@ -65,9 +65,10 @@ jobs:
       - name: Obtain the changelog
         id: generate_changelog
         run: |
+          poe docs
           {
             echo 'changelog<<EOF'
-            poe changelog
+            poe --quiet changelog
             echo EOF
           } >> "$GITHUB_OUTPUT"
 

extra/release.py

@@ -6,22 +6,141 @@
 
 import re
 import subprocess
+from contextlib import redirect_stdout
 from datetime import datetime, timezone
+from functools import partial
+from io import StringIO
 from pathlib import Path
-from typing import Callable
+from typing import Callable, NamedTuple
 
 import click
 import tomli
 from packaging.version import Version, parse
+from sphinx.ext import intersphinx
+from typing_extensions import TypeAlias
 
 BASE = Path(__file__).parent.parent.absolute()
 PYPROJECT = BASE / "pyproject.toml"
 CHANGELOG = BASE / "docs" / "changelog.rst"
+DOCS = "https://beets.readthedocs.io/en/stable"
 
-MD_CHANGELOG_SECTION_LIST = re.compile(r"- .+?(?=\n\n###|$)", re.DOTALL)
-version_header = r"\d+\.\d+\.\d+ \([^)]+\)"
+VERSION_HEADER = r"\d+\.\d+\.\d+ \([^)]+\)"
 RST_LATEST_CHANGES = re.compile(
-    rf"{version_header}\n--+\s+(.+?)\n\n+{version_header}", re.DOTALL
+    rf"{VERSION_HEADER}\n--+\s+(.+?)\n\n+{VERSION_HEADER}", re.DOTALL
+)
+
+Replacement: TypeAlias = "tuple[str, str | Callable[[re.Match[str]], str]]"
+
+
+class Ref(NamedTuple):
+    """A reference to documentation with ID, path, and optional title."""
+
+    id: str
+    path: str | None
+    title: str | None
+
+    @classmethod
+    def from_line(cls, line: str) -> Ref:
+        """Create Ref from a Sphinx objects.inv line.
+
+        Each line has the following structure:
+        <id>    [optional title : ] <relative-url-path>
+
+        """
+        if len(line_parts := line.split(" ", 1)) == 1:
+            return cls(line, None, None)
+
+        id, path_with_name = line_parts
+        parts = [p.strip() for p in path_with_name.split(":", 1)]
+
+        if len(parts) == 1:
+            path, name = parts[0], None
+        else:
+            name, path = parts
+
+        return cls(id, path, name)
+
+    @property
+    def url(self) -> str:
+        """Full documentation URL."""
+        return f"{DOCS}/{self.path}"
+
+    @property
+    def name(self) -> str:
+        """Display name (title if available, otherwise ID)."""
+        return self.title or self.id
+
+
+def get_refs() -> dict[str, Ref]:
+    """Parse Sphinx objects.inv and return dict of documentation references."""
+    objects_filepath = Path("docs/_build/html/objects.inv")
+    if not objects_filepath.exists():
+        raise ValueError("Documentation does not exist. Run 'poe docs' first.")
+
+    captured_output = StringIO()
+
+    with redirect_stdout(captured_output):
+        intersphinx.inspect_main([str(objects_filepath)])
+
+    return {
+        r.id: r
+        for ln in captured_output.getvalue().split("\n")
+        if ln.startswith("\t") and (r := Ref.from_line(ln.strip()))
+    }
+
+
+def create_rst_replacements() -> list[Replacement]:
+    """Generate list of pattern replacements for RST changelog."""
+    refs = get_refs()
+
+    def make_ref_link(ref_id: str, name: str | None = None) -> str:
+        ref = refs[ref_id]
+        return rf"`{name or ref.name} <{ref.url}>`_"
+
+    commands = "|".join(r.split("-")[0] for r in refs if r.endswith("-cmd"))
+    plugins = "|".join(
+        r.split("/")[-1] for r in refs if r.startswith("plugins/")
+    )
+    return [
+        # Fix nested bullet points indent: use 2 spaces consistently
+        (r"(?<=\n) {3,4}(?=\*)", "  "),
+        # Fix nested text indent: use 4 spaces consistently
+        (r"(?<=\n) {5,6}(?=[\w:`])", "    "),
+        # Replace Sphinx :ref: and :doc: directives by documentation URLs
+        #   :ref:`/plugins/autobpm` -> [AutoBPM Plugin](DOCS/plugins/autobpm.html)
+        (
+            r":(?:ref|doc):`+(?:([^`<]+)<)?/?([\w./_-]+)>?`+",
+            lambda m: make_ref_link(m[2], m[1]),
+        ),
+        # Convert command references to documentation URLs
+        #   `beet move` or `move` command -> [import](DOCS/reference/cli.html#import)
+        (
+            rf"`+beet ({commands})`+|`+({commands})`+(?= command)",
+            lambda m: make_ref_link(f"{m[1] or m[2]}-cmd"),
+        ),
+        # Convert plugin references to documentation URLs
+        #   `fetchart` plugin -> [fetchart](DOCS/plugins/fetchart.html)
+        (rf"`+({plugins})`+", lambda m: make_ref_link(f"plugins/{m[1]}")),
+        # Add additional backticks around existing backticked text to ensure it
+        # is rendered as inline code in Markdown
+        (r"(?<=[\s])(`[^`]+`)(?!_)", r"`\1`"),
+        # Convert bug references to GitHub issue links
+        (r":bug:`(\d+)`", r":bug: (#\1)"),
+        # Convert user references to GitHub @mentions
+        (r":user:`(\w+)`", r"\@\1"),
+    ]
+
+
+MD_REPLACEMENTS: list[Replacement] = [
+    (r"^  (- )", r"\1"),  # remove indent from top-level bullet points
+    (r"^ +(  - )", r"\1"),  # adjust nested bullet points indent
+    (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"),  # format section headers
+    (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"),  # and bolden too long ones
+    (r"### [^\n]+\n+(?=### )", ""),  # remove empty sections
+]
+order_bullet_points = partial(
+    re.compile("(\n- .*?(?=\n(?! *- )|$))", flags=re.DOTALL).sub,
+    lambda m: "\n- ".join(sorted(m.group().split("\n- "))),
 )
 
 
@@ -41,8 +160,11 @@ def update_changelog(text: str, new: Version) -> str:
 ----------
 
 New features:
+
 Bug fixes:
+
 For packagers:
+
 Other changes:
 
 {new_header}
@@ -95,50 +217,36 @@ def bump_version(new: Version) -> None:
 
 def rst2md(text: str) -> str:
     """Use Pandoc to convert text from ReST to Markdown."""
-    # Other backslashes with verbatim ranges.
-    rst = re.sub(r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``", text)
-
-    # Bug numbers.
-    rst = re.sub(r":bug:`(\d+)`", r":bug: (#\1)", rst)
-
-    # Users.
-    rst = re.sub(r":user:`(\w+)`", r"@\1", rst)
     return (
         subprocess.check_output(
-            ["/usr/bin/pandoc", "--from=rst", "--to=gfm", "--wrap=none"],
-            input=rst.encode(),
+            ["pandoc", "--from=rst", "--to=gfm+hard_line_breaks"],
+            input=text.encode(),
         )
         .decode()
         .strip()
     )
 
 
-def changelog_as_markdown() -> str:
-    """Get the latest changelog entry as hacked up Markdown."""
-    with CHANGELOG.open() as f:
-        contents = f.read()
+def get_changelog_contents() -> str | None:
+    if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()):
+        return m.group(1)
 
-    m = RST_LATEST_CHANGES.search(contents)
-    rst = m.group(1) if m else ""
+    return None
 
-    # Convert with Pandoc.
-    md = rst2md(rst)
 
-    # Make sections stand out
-    md = re.sub(r"^(\w.+?):$", r"### \1", md, flags=re.M)
+def changelog_as_markdown(rst: str) -> str:
+    """Get the latest changelog entry as hacked up Markdown."""
+    for pattern, repl in create_rst_replacements():
+        rst = re.sub(pattern, repl, rst, flags=re.M | re.DOTALL)
 
-    # Highlight plugin names
-    md = re.sub(
-        r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:", md, flags=re.M
-    )
+    md = rst2md(rst)
 
-    # Highlights command names.
-    md = re.sub(r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:", md, flags=re.M)
+    for pattern, repl in MD_REPLACEMENTS:
+        md = re.sub(pattern, repl, md, flags=re.M | re.DOTALL)
 
-    # sort list items alphabetically for each of the sections
-    return MD_CHANGELOG_SECTION_LIST.sub(
-        lambda m: "\n".join(sorted(m.group().splitlines())), md
-    )
+    # order bullet points in each of the lists alphabetically to
+    # improve readability
+    return order_bullet_points(md)
 
 
 @click.group()
@@ -156,7 +264,11 @@ def bump(version: Version) -> None:
 @cli.command()
 def changelog():
     """Get the most recent version's changelog as Markdown."""
-    print(changelog_as_markdown())
+    if changelog := get_changelog_contents():
+        try:
+            print(changelog_as_markdown(changelog))
+        except ValueError as e:
+            raise click.exceptions.UsageError(str(e))
 
 
 if __name__ == "__main__":

test/test_release.py

@@ -0,0 +1,108 @@
+"""Tests for the release utils."""
+
+import os
+import shutil
+import sys
+
+import pytest
+
+release = pytest.importorskip("extra.release")
+
+
+pytestmark = pytest.mark.skipif(
+    not (
+        (os.environ.get("GITHUB_ACTIONS") == "true" and sys.platform != "win32")
+        or bool(shutil.which("pandoc"))
+    ),
+    reason="pandoc isn't available",
+)
+
+
+@pytest.fixture
+def rst_changelog():
+    return """New features:
+
+* :doc:`/plugins/substitute`: Some substitute
+  multi-line change.
+  :bug:`5467`
+* :ref:`list-cmd` Update.
+
+You can do something with this command::
+
+    $ do-something
+
+Bug fixes:
+
+* Some fix that refers to an issue.
+  :bug:`5467`
+* Some fix that mentions user :user:`username`.
+* Some fix thanks to
+  :user:`username`. :bug:`5467`
+* Some fix with its own bullet points using incorrect indentation:
+   * First nested bullet point
+     with some text that wraps to the next line
+   * Second nested bullet point
+* Another fix with its own bullet points using correct indentation:
+  * First
+  * Second
+
+Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80
+characters:
+
+Empty section:
+
+Other changes:
+
+* Changed `bitesize` label to `good first issue`. Our `contribute`_ page is now
+  automatically populated with these issues. :bug:`4855`
+
+.. _contribute: https://github.com/beetbox/beets/contribute
+
+2.1.0 (November 22, 2024)
+-------------------------
+
+Bug fixes:
+
+* Fixed something."""
+
+
+@pytest.fixture
+def md_changelog():
+    return r"""### New features
+
+- [Substitute Plugin](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467)
+- [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list-cmd) Update.
+
+You can do something with this command:
+
+    $ do-something
+
+### Bug fixes
+
+- Another fix with its own bullet points using correct indentation:
+  - First
+  - Second
+- Some fix thanks to @username. :bug: (\#5467)
+- Some fix that mentions user @username.
+- Some fix that refers to an issue. :bug: (\#5467)
+- Some fix with its own bullet points using incorrect indentation:
+  - First nested bullet point with some text that wraps to the next line
+  - Second nested bullet point
+
+**Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80 characters**
+
+### Other changes
+
+- Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855)
+
+# 2.1.0 (November 22, 2024)
+
+### Bug fixes
+
+- Fixed something."""  # noqa: E501
+
+
+def test_convert_rst_to_md(rst_changelog, md_changelog):
+    actual = release.changelog_as_markdown(rst_changelog)
+
+    assert actual == md_changelog