⬆️ Allow docutils 0.22

Author: chrisjsewell

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        docutils-version: ["0.20", "0.21"]
+        docutils-version: ["0.20", "0.21", "0.22"]
 
     steps:
     - name: Checkout source

pyproject.toml

@@ -34,7 +34,7 @@ keywords = [
 ]
 requires-python = ">=3.11"
 dependencies = [
-    "docutils>=0.20,<0.22",
+    "docutils>=0.20,<0.23",
     "jinja2", # required for substitutions, but let sphinx choose version
     "markdown-it-py~=4.0",
     "mdit-py-plugins~=0.5",
@@ -146,6 +146,7 @@ disallow_any_generics = false
 [tool.pytest.ini_options]
 filterwarnings = [
     "ignore:.*The default for the setting.*:FutureWarning",
+    "ignore:.*:PendingDeprecationWarning",
 ]
 
 [tool.coverage.run]

tests/conftest.py

@@ -0,0 +1,55 @@
+"""Shared pytest configuration for all tests."""
+
+import re
+
+from docutils import __version_info__ as docutils_version_info
+
+# docutils 0.22+ serializes boolean attributes as "1"/"0" instead of "True"/"False"
+DOCUTILS_0_22_PLUS = docutils_version_info >= (0, 22)
+
+
+def normalize_doctree_xml(text: str) -> str:
+    """Normalize docutils XML output for cross-version compatibility.
+
+    In docutils 0.22+, boolean attributes are serialized as "1"/"0"
+    instead of "True"/"False". This function normalizes to the new format
+    for consistent test fixtures.
+    """
+    if DOCUTILS_0_22_PLUS:
+        # Normalize new format (True/False) to old format (1/0)
+        # Only replace when it's clearly a boolean attribute value
+        # Pattern: attribute="True" or attribute="False"
+        attrs = [
+            "force",
+            "glob",
+            "hidden",
+            "id_link",
+            "includehidden",
+            "inline",
+            "internal",
+            "is_div",
+            "linenos",
+            "multi_line_parameter_list",
+            "multi_line_trailing_comma",
+            "no-contents-entry",
+            "no-index",
+            "no-index-entry",
+            "no-typesetting",
+            "no-wrap",
+            "nocontentsentry",
+            "noindex",
+            "noindexentry",
+            "nowrap",
+            "refexplicit",
+            "refspecific",
+            "refwarn",
+            "sorted",
+            "titlesonly",
+            "toctree",
+            "translatable",
+        ]
+        text = re.sub(rf' ({"|".join(attrs)})="1"', r' \1="True"', text)
+        text = re.sub(rf' ({"|".join(attrs)})="0"', r' \1="False"', text)
+        # numbered is changed in math_block, but not in toctree, so we have to be more precise
+        text = re.sub(r' numbered="1" xml:space', r' numbered="True" xml:space', text)
+    return text

tests/test_html/test_html_to_nodes.py

@@ -6,6 +6,7 @@
 
 from myst_parser.config.main import MdParserConfig
 from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes
+from tests.conftest import normalize_doctree_xml
 
 FIXTURE_PATH = Path(__file__).parent
 
@@ -32,4 +33,4 @@ def _run_directive(name: str, first_line: str, content: str, position: int):
 def test_html_to_nodes(file_params, mock_renderer):
     output = nodes.container()
     output += html_to_nodes(file_params.content, line_number=0, renderer=mock_renderer)
-    file_params.assert_expected(output.pformat(), rstrip=True)
+    file_params.assert_expected(normalize_doctree_xml(output.pformat()), rstrip=True)

tests/test_renderers/test_fixtures_docutils.py

@@ -11,6 +11,7 @@
 from typing import Any
 
 import pytest
+from conftest import normalize_doctree_xml
 from docutils.core import Publisher, publish_doctree
 from pytest_param_files import ParamTestData
 
@@ -37,7 +38,9 @@ def _apply_transforms(self):
     )
 
     # in docutils 0.18 footnote ids have changed
-    outcome = doctree.pformat().replace('"footnote-reference-1"', '"id1"')
+    outcome = normalize_doctree_xml(doctree.pformat()).replace(
+        '"footnote-reference-1"', '"id1"'
+    )
     outcome = outcome.replace(' language=""', "")
     file_params.assert_expected(outcome, rstrip_lines=True)
 
@@ -54,7 +57,7 @@ def test_link_resolution(file_params: ParamTestData):
         parser=Parser(),
         settings_overrides=settings,
     )
-    outcome = doctree.pformat()
+    outcome = normalize_doctree_xml(doctree.pformat())
     if report_stream.getvalue().strip():
         outcome += "\n\n" + report_stream.getvalue().strip()
     file_params.assert_expected(outcome, rstrip_lines=True)
@@ -75,7 +78,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_directives.md")
@@ -95,7 +100,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_extensions.txt")
@@ -109,7 +116,9 @@ def test_syntax_extensions(file_params: ParamTestData):
         parser=Parser(),
         settings_overrides=settings,
     )
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 def settings_from_cmdline(cmdline: str | None) -> dict[str, Any]:

tests/test_renderers/test_fixtures_sphinx.py

@@ -17,6 +17,7 @@
 from sphinx_pytest.plugin import CreateDoctree
 
 from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
+from tests.conftest import normalize_doctree_xml
 
 FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures")
 
@@ -36,7 +37,7 @@ def _apply_transforms(self):
         monkeypatch.setattr(SphinxTransformer, "apply_transforms", _apply_transforms)
 
     result = sphinx_doctree(file_params.content, "index.md")
-    pformat = result.pformat("index")
+    pformat = normalize_doctree_xml(result.pformat("index"))
     replacements = {
         # changed in docutils 0.20.1
         '<literal classes="code" language="">': '<literal classes="code">',
@@ -58,7 +59,7 @@ def test_link_resolution(file_params: ParamTestData, sphinx_doctree: CreateDoctr
     sphinx_doctree.srcdir.joinpath("test.txt").touch()
     sphinx_doctree.srcdir.joinpath("other.rst").write_text(":orphan:\n\nTest\n====")
     result = sphinx_doctree(file_params.content, "index.md")
-    outcome = result.pformat("index")
+    outcome = normalize_doctree_xml(result.pformat("index"))
     if result.warnings.strip():
         outcome += "\n\n" + result.warnings.strip()
     file_params.assert_expected(outcome, rstrip_lines=True)
@@ -80,7 +81,9 @@ def settings_from_json(string: str | None):
 def test_tables(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoctree):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "directive_options.md")
@@ -89,7 +92,9 @@ def test_directive_options(
 ):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "sphinx_directives.md")
@@ -102,7 +107,9 @@ def test_sphinx_directives(
         pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
-    pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    pformat = normalize_doctree_xml(
+        sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    )
     # see https://github.com/executablebooks/MyST-Parser/issues/522
     if sys.maxsize == 2147483647:
         pformat = pformat.replace('"2147483647"', '"9223372036854775807"')
@@ -115,7 +122,9 @@ def test_sphinx_roles(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDo
         pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
-    pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    pformat = normalize_doctree_xml(
+        sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    )
     # sphinx 3 adds a parent key
     pformat = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', pformat)
     # sphinx >= 4.5.0 adds a trailing slash to PEP URLs,
@@ -136,7 +145,9 @@ def test_dollarmath(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["dollarmath"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "amsmath.md")
@@ -148,7 +159,9 @@ def test_amsmath(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["amsmath"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "containers.md")
@@ -160,7 +173,9 @@ def test_containers(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["colon_fence"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "eval_rst.md")
@@ -169,7 +184,9 @@ def test_evalrst_elements(
 ):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "definition_lists.md")
@@ -180,7 +197,9 @@ def test_definition_lists(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["deflist"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "attributes.md")
@@ -192,4 +211,6 @@ def test_attributes(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct
         }
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )

tests/test_renderers/test_include_directive.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import pytest
+from conftest import normalize_doctree_xml
 from docutils.core import publish_doctree
 
 from myst_parser.parsers.docutils_ import Parser
@@ -24,7 +25,11 @@ def test_render(file_params, tmp_path, monkeypatch):
     )
 
     doctree["source"] = "tmpdir/test.md"
-    output = doctree.pformat().replace(str(tmp_path) + os.sep, "tmpdir/").rstrip()
+    output = (
+        normalize_doctree_xml(doctree.pformat())
+        .replace(str(tmp_path) + os.sep, "tmpdir/")
+        .rstrip()
+    )
 
     file_params.assert_expected(output, rstrip=True)
 

tests/test_renderers/test_myst_config.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import pytest
+from conftest import normalize_doctree_xml
 from docutils.core import Publisher, publish_string
 from pytest_param_files import ParamTestData
 
@@ -36,6 +37,7 @@ def test_cmdline(file_params: ParamTestData):
         writer_name="pseudoxml",
         settings_overrides=settings,
     )
+    output = normalize_doctree_xml(output)
     warnings = report_stream.getvalue()
     if warnings:
         output += "\n" + warnings

tests/test_renderers/test_myst_refs.py

@@ -1,6 +1,7 @@
 import sys
 
 import pytest
+from conftest import normalize_doctree_xml
 from sphinx.util.console import strip_colors
 from sphinx_pytest.plugin import CreateDoctree
 
@@ -55,7 +56,7 @@ def test_parse(
 
     doctree["source"] = "root/index.md"
     doctree.attributes.pop("translation_progress", None)
-    outcome = doctree.pformat()
+    outcome = normalize_doctree_xml(doctree.pformat())
     if result.warnings.strip():
         outcome += "\n\n" + strip_colors(result.warnings.strip())
     file_regression.check(outcome, basename=test_name, extension=".xml")

tests/test_sphinx/conftest.py

@@ -41,6 +41,7 @@ def test_basic(app, status, warning, get_sphinx_app_output):
 from docutils import nodes
 
 from myst_parser._compat import findall
+from tests.conftest import normalize_doctree_xml
 
 SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs"))
 
@@ -125,7 +126,7 @@ def read(
             node.attributes.pop("translated", None)
 
         if regress:
-            text = doctree.pformat()  # type: str
+            text = normalize_doctree_xml(doctree.pformat())  # type: str
             for find, rep in (replace or {}).items():
                 text = text.replace(find, rep)
             if rstrip_lines: