generate_packages_doc.py

# SPDX-FileCopyrightText: 2025 BayLibre, SAS
# SPDX-License-Identifier: Apache-2.0

import glob
import os
import re

import packaging.version as packaging_version
import requests
import yaml


GITLAB_PROJECT_ID = "56254198"
GITLAB_REGISTRY_URL = (
    f"https://gitlab.com/api/v4/projects/{GITLAB_PROJECT_ID}/packages"
)
GITLAB_WHEEL_BUILDER_URL = (
    "https://gitlab.com/riseproject/python/wheel_builder/-/tree/main"
)
PYPI_INDEX_URL = "https://pypi.riseproject.dev/simple/"

# Match RST inline external refs: `Label <url>`_ or `Label <url>`__
RST_LINK_RE = re.compile(r"`([^`<]+?)\s*<([^>]+)>`_+")


def rst_to_md(text):
    """Convert the subset of RST inline syntax used in YAML comments to Markdown."""
    if not text:
        return text
    return RST_LINK_RE.sub(r"[\1](\2)", text)


def get_all_packages():
    """Fetch all package data from GitLab, handling pagination."""
    session = requests.Session()
    packages = []
    page = 1

    try:
        while True:
            response = session.get(
                GITLAB_REGISTRY_URL, params={"per_page": 100, "page": page}
            )
            if response.status_code != 200:
                print(
                    f"Error fetching packages: {response.status_code} - {response.text}"
                )
                return []

            data = response.json()
            if not data:
                break

            packages.extend(data)
            next_page = response.headers.get("X-Next-Page")
            if not next_page:
                break
            page = int(next_page)

    except requests.RequestException as e:
        print(f"Request error: {e}")
        return []

    return packages


def get_package_id(package_name, version, package_list):
    return next(
        (
            pkg["id"]
            for pkg in package_list
            if pkg["name"] == package_name and pkg["version"].startswith(version)
        ),
        None,
    )


def get_upstream_tag(package_name, version, package_list):
    return next(
        (
            pkg["pipeline"]["ref"]
            for pkg in package_list
            if pkg["name"] == package_name
            and pkg["version"].startswith(version)
            and "pipeline" in pkg
        ),
        None,
    )


def _callout(lines, label, text):
    """Append a just-the-docs callout block to ``lines``."""
    lines.append("")
    lines.append(f"{{: .{label} }}")
    for ln in rst_to_md(text).splitlines():
        lines.append(f"> {ln}" if ln else ">")
    lines.append("")


def generate_md_page(yaml_file, output_md, package_list):
    """Generate a Markdown page from a single package YAML file."""
    try:
        with open(yaml_file, "r", encoding="utf-8") as f:
            package_data = yaml.safe_load(f)
    except (yaml.YAMLError, FileNotFoundError) as e:
        print(f"Error reading {yaml_file}: {e}")
        return None, None

    package_name = package_data.get("package-name", "Unknown Package")
    license_type = package_data.get("license")
    source_code = package_data.get("source-code")
    comment = package_data.get("comment")
    warning = package_data.get("warning")
    versions = package_data.get("versions", [])

    versions.sort(
        key=lambda v: packaging_version.parse(str(v["version"])), reverse=True
    )
    latest_version = str(versions[0]["version"]) if versions else None

    lines = [
        "---",
        f"title: {package_name}",
        "layout: default",
        "parent: Supported Packages",
        "---",
        "",
        "<!-- Auto-generated by generate_packages_doc.py. Do not edit manually. -->",
        "",
        f"# {package_name}",
        "",
    ]

    if "deprecated" in package_data:
        pypi_url = f"https://pypi.org/project/{package_name}/"
        lines += [
            "{: .warning }",
            "> This package is deprecated. PyPI now publishes newer versions of",
            "> this package for riscv64, and we will no longer maintain this",
            f"> package. Please use the version from [PyPI]({pypi_url}) instead.",
            ">",
            "> If you need a specific version between the latest available here",
            "> and the first available on PyPI, please open an",
            "> [Issue](https://gitlab.com/riseproject/python/wheel_builder/-/issues).",
            "",
        ]

    if source_code:
        lines.append(f"- **Source Code:** [{source_code}]({source_code})")

    lines.append("- **Supported versions:**")
    lines.append("")

    for version in versions:
        version_number = str(version["version"])
        is_latest = version_number == latest_version
        summary_label = f"{version_number} (latest)" if is_latest else version_number

        lines.append(
            f'<details markdown="1"{" open" if is_latest else ""}>'
        )
        lines.append(f"<summary><strong>{summary_label}</strong></summary>")
        lines.append("")

        install_command = (
            f"pip install {package_name} --index-url {PYPI_INDEX_URL}"
            if is_latest
            else f"pip install {package_name}=={version_number} "
            f"--index-url {PYPI_INDEX_URL}"
        )
        lines += ["```bash", install_command, "```", ""]

        lic = version.get("license", license_type)
        if lic:
            lines.append(f"- **License:** {lic}")

        package_id = get_package_id(package_name, version_number, package_list)
        if package_id:
            registry_link = (
                f"https://gitlab.com/riseproject/python/wheel_builder/-/packages/{package_id}"
            )
            lines.append(
                f"- **Download files:** [{registry_link}]({registry_link})"
            )

        if "patched" in version and source_code:
            project_name = (
                source_code.removesuffix("/").removesuffix(".git").split("/")[-1]
            )
            upstream_tag = get_upstream_tag(
                package_name, version_number, package_list
            )
            if upstream_tag:
                patch_link = (
                    f"{GITLAB_WHEEL_BUILDER_URL}/wheel_builder/"
                    f"{project_name}/patches/{upstream_tag}"
                )
                lines.append(
                    f"- **Patch applied for this version:** "
                    f"[{patch_link}]({patch_link})"
                )

        if version.get("comment"):
            _callout(lines, "note", version["comment"])

        if version.get("warning"):
            _callout(lines, "warning", version["warning"])

        lines += ["</details>", ""]

    if comment:
        _callout(lines, "note", comment)

    if warning:
        _callout(lines, "warning", warning)

    try:
        with open(output_md, "w", encoding="utf-8") as f:
            f.write("\n".join(lines).rstrip() + "\n")
    except IOError as e:
        print(f"Error writing {output_md}: {e}")
        return None, None

    return package_name, output_md


def process_all_yaml_files(package_list):
    out_dir = os.path.dirname(os.path.abspath(__file__))
    yaml_files = sorted(glob.glob(os.path.join(out_dir, "*.yaml")))
    if not yaml_files:
        print(f"No YAML files found in {out_dir}")
        return

    package_entries = []
    for yaml_file in yaml_files:
        base = os.path.basename(yaml_file).replace(".yaml", ".md")
        md_file = os.path.join(out_dir, base)
        name, fname = generate_md_page(yaml_file, md_file, package_list)
        if name and fname:
            package_entries.append((name, os.path.basename(fname)))

    package_entries.sort(key=lambda x: x[0].lower())
    generate_index(package_entries, out_dir)


def generate_index(package_entries, out_dir):
    if not package_entries:
        print("No packages to list in index.md")
        return

    lines = [
        "---",
        "title: Supported Packages",
        "layout: default",
        "nav_order: 5",
        "has_children: true",
        "---",
        "",
        "<!-- Auto-generated by generate_packages_doc.py. Do not edit manually. -->",
        "",
        "# List of Supported Packages",
        "",
    ]
    for name, fname in package_entries:
        page = fname.replace(".md", ".html")
        lines.append(f"- [{name}]({page})")

    index_path = os.path.join(out_dir, "index.md")
    try:
        with open(index_path, "w", encoding="utf-8") as f:
            f.write("\n".join(lines) + "\n")
        print(f"Generated: {index_path}")
    except IOError as e:
        print(f"Error writing {index_path}: {e}")


if __name__ == "__main__":
    package_list = get_all_packages()
    process_all_yaml_files(package_list)