replace m2r (deprecated, vulnerabilities)

Author: geritwagner

colrev/package_manager/doc_registry_manager.py

@@ -5,6 +5,7 @@
 
 import json
 import os
+import re
 import tempfile
 import typing
 from pathlib import Path
@@ -13,7 +14,6 @@
 import requests
 import toml
 from bs4 import BeautifulSoup
-from m2r import parse_from_file  # pylint: disable=import-error
 
 import colrev.package_manager.colrev_internal_packages
 from colrev.constants import EndpointType
@@ -271,18 +271,83 @@ def _get_header_info(self) -> str:
 
     def import_package_docs(self) -> None:
         """Import the package documentation."""
-        with open(
-            Filepaths.COLREV_PATH
-            / Path(f"docs/source/manual/packages/{self.docs_rst_path}"),
-            "w",
-            encoding="utf-8",
-        ) as file:
+        docs_package_path = Filepaths.COLREV_PATH / Path(
+            f"docs/source/manual/packages/{self.docs_rst_path}"
+        )
+
+        readme_path = self._copy_readme_for_myst_include()
+        readme_content = readme_path.read_text(encoding="utf-8")
+        rst_readme_content = self._convert_markdown_to_rst(readme_content)
+
+        with open(docs_package_path, "w", encoding="utf-8") as file:
             file.write(self._get_header_info())
-            output = parse_from_file(self.docs_package_readme_path)
-            output = output.replace(
-                ".. list-table::", ".. list-table::\n   :align: left"
-            )
-            file.write(output)
+            file.write("\n")
+            file.write(rst_readme_content)
+
+    def _convert_markdown_to_rst(self, markdown_content: str) -> str:
+        """Convert Markdown content to reStructuredText for generated docs pages."""
+        heading_chars = {1: "=", 2: "-", 3: "^", 4: '"'}
+        converted_lines = []
+        in_fenced_code_block = False
+        fenced_code_language = ""
+
+        for line in markdown_content.splitlines():
+            fence_match = re.match(r"^```(\w+)?\s*$", line)
+            if fence_match:
+                if not in_fenced_code_block:
+                    in_fenced_code_block = True
+                    fenced_code_language = fence_match.group(1) or ""
+                    if fenced_code_language:
+                        converted_lines.append(
+                            f".. code-block:: {fenced_code_language}"
+                        )
+                    else:
+                        converted_lines.append(".. code-block::")
+                    converted_lines.append("")
+                else:
+                    in_fenced_code_block = False
+                    fenced_code_language = ""
+                    converted_lines.append("")
+                continue
+
+            if in_fenced_code_block:
+                if line.strip() == "":
+                    converted_lines.append("")
+                else:
+                    converted_lines.append(f"   {line}")
+                continue
+
+            heading_match = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
+            if heading_match:
+                level = len(heading_match.group(1))
+                title = heading_match.group(2)
+                underline_char = heading_chars.get(level, '"')
+                converted_lines.append(title)
+                converted_lines.append(underline_char * len(title))
+                converted_lines.append("")
+                continue
+
+            line = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r"`\1 <\2>`_", line)
+            line = re.sub(r"^\s*-\s+`([^`]+)\s+<([^>]+)>`_\s*$", r"* `\1 <\2>`_", line)
+            line = re.sub(r"``([^`]+)```\1`", r"``\1``", line)
+            converted_lines.append(line)
+
+        return "\n".join(converted_lines).rstrip() + "\n"
+
+    def _copy_readme_for_myst_include(self) -> Path:
+        """Copy the package README to a generated docs folder for MyST inclusion."""
+        readme_dir = Filepaths.COLREV_PATH / Path(
+            "docs/source/manual/packages/_package_readmes"
+        )
+        readme_dir.mkdir(parents=True, exist_ok=True)
+
+        readme_path = readme_dir / f"{self.package_id}.md"
+        readme_path.write_text(
+            self.docs_package_readme_path.read_text(encoding="utf-8"),
+            encoding="utf-8",
+        )
+
+        return readme_path
 
     def get_endpoint_item(self, endpoint_type: EndpointType) -> dict:
         """Get the endpoint item for the package.

docs/source/conf.py

@@ -50,6 +50,7 @@
     "sphinx_collapse",
     "sphinx_design",
     "sphinx_copybutton",
+    "myst_parser",
 ]
 
 
@@ -64,6 +65,7 @@
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
+exclude_patterns = ["manual/packages/_package_readmes/**"]
 
 # -- Options for HTML output -------------------------------------------------
 

docs/source/manual/packages/colrev-sync.rst

@@ -46,14 +46,16 @@ colrev-sync
 Summary
 -------
 
+
 The colrev-sync package imports references from CoLRev projects (through local_index) into non-CoLRev paper projects that use Markdown and BibTeX.
 If BibTeX citations keys are used in the paper project, the following command retrieves the corresponding bibliographical details and adds them to the BibTeX file:
 
 .. code-block::
 
    colrev-sync
 
-CoLRev sync can also be used through pre-commit hooks, when the following is included in the ``.pre-commit-config.yaml``\ :
+
+CoLRev sync can also be used through pre-commit hooks, when the following is included in the `.pre-commit-config.yaml`:
 
 .. code-block::
 

docs/source/manual/packages/colrev.abi_inform_proquest.rst

@@ -51,12 +51,15 @@ colrev.abi_inform_proquest
 Summary
 -------
 
+
 search
 ------
 
+
 DB search
 ^^^^^^^^^
 
+
 `ABI/INFORM ProQuest <https://about.proquest.com/en/products-services/abi_inform_complete/>`_
 
 Links

docs/source/manual/packages/colrev.acm_digital_library.rst

@@ -51,16 +51,20 @@ colrev.acm_digital_library
 Summary
 -------
 
+
 search
 ------
 
+
 DB search
 ^^^^^^^^^
 
-.. code-block::
+
+.. code-block:: bash
 
    colrev search --add colrev.acm_digital_library
 
+
 Links
 -----
 

docs/source/manual/packages/colrev.add_journal_ranking.rst

@@ -51,9 +51,11 @@ colrev.add_journal_ranking
 Summary
 -------
 
+
 prep
 ----
 
+
 **Prerequisite** Initial ranking data is extracted from ranking.csv into SQLite Database sqlite_index.db with 'colrev env -i'.
 
 **Description**
@@ -62,19 +64,21 @@ The add_journal_ranking package allows the user to add a ranking to the records
 
 Example:
 
-.. code-block::
+.. code-block:: text
 
    journal_ranking = {Senior Scholars' List of Premier Journals}, or
    journal_ranking = {not included in a ranking},
 
+
 Should the journal be in the Beall's Predatory Journal list, then the record will be marked as "Predatory Journal: Do not include!" and be predestined to be excluded in the scope_prescreen process.
 
 Example:
 
-.. code-block::
+.. code-block:: text
 
    journal_ranking = {Predatory Journal: Do not include!},
 
+
 The journal ranking will also be used in the colrev prescreen method and allows the user to decide if the record should be marked as 'rev_prescreen_excluded' or 'rev_prescreen_included'.
 
 For further information see `scope_prescreen <https://colrev-environment.github.io/colrev/manual/packages/colrev.scope_prescreen.html>`_.

docs/source/manual/packages/colrev.ais_library.rst

@@ -51,37 +51,44 @@ colrev.ais_library
 Summary
 -------
 
+
 search
 ------
 
-**Limitation**\ : The AIS eLibrary currently limits search results an 3.000 records (for DB and API searches).
+
+**Limitation**: The AIS eLibrary currently limits search results an 3.000 records (for DB and API searches).
 
 DB search
 ^^^^^^^^^
 
+
 Run a search on `aisel.aisnet.org <https://aisel.aisnet.org/>`_.
 
-Download the search results (advanced search, format:Bibliography Export, click Search) and store them in the ``data/search/`` directory.
+Download the search results (advanced search, format:Bibliography Export, click Search) and store them in the `data/search/` directory.
 
-.. code-block::
+.. code-block:: bash
 
    colrev search --add colrev.ais_library
 
+
 API search
 ^^^^^^^^^^
 
+
 Copy the search link and add an API search (replacing the link):
 
-.. code-block::
+.. code-block:: bash
 
    colrev search --add colrev.ais_library -p "https://aisel.aisnet.org/do/search/?q=microsourcing&start=0&context=509156&facet="
 
+
 Note: Complex queries can be entered in the basic search field. Example:
 
-.. code-block::
+.. code-block:: text
 
    title:microsourcing AND ( digital OR online)
 
+
 Format of the search-history file (DB search):
 
 .. code-block:: json
@@ -94,6 +101,7 @@ Format of the search-history file (DB search):
        "version": "0.1.0"
    }
 
+
 Format of the search-history file (API search):
 
 .. code-block:: json
@@ -109,7 +117,9 @@ Format of the search-history file (API search):
        "version": "0.1.0"
    }
 
+
 Links
 -----
 
+
 `AIS eLibrary <https://aisel.aisnet.org/>`_

docs/source/manual/packages/colrev.arxiv.rst

@@ -51,18 +51,22 @@ colrev.arxiv
 Summary
 -------
 
+
 Add the search source
 ---------------------
 
+
 API
 ^^^
 
+
 ℹ️ Restriction: API searches do not support complex queries (yet)
 
-.. code-block::
+.. code-block:: bash
 
    colrev search --add colrev.arxiv -p "https://arxiv.org/search/?query=fitbit&searchtype=all&abstracts=show&order=-announced_date_first&size=50"
 
+
 Format of the search-history file:
 
 .. code-block:: json
@@ -78,6 +82,7 @@ Format of the search-history file:
        "version": "0.1.0"
    }
 
+
 Links
 -----
 

docs/source/manual/packages/colrev.bibliography_export.rst

@@ -51,14 +51,16 @@ colrev.bibliography_export
 Summary
 -------
 
+
 data
 ----
 
+
 This endpoint exports the records in different bibliographical formats, which can be useful when the team works with a particular reference manager.
 
 To add an endpoint, run any of the following:
 
-.. code-block::
+.. code-block:: bash
 
    colrev data -a colrev.bibliography_export -p zotero
    colrev data -a colrev.bibliography_export -p jabref
@@ -69,6 +71,4 @@ To add an endpoint, run any of the following:
    colrev data -a colrev.bibliography_export -p EXCEL
 
 
-.. raw:: html
-
-   <!-- ## Links -->
+<!-- ## Links -->

docs/source/manual/packages/colrev.blank.rst

@@ -51,4 +51,5 @@ colrev.blank
 Summary
 -------
 
+
 The *blank* ReviewType does not include any settings. It is primarily used for simulations.