diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 4648547..9b6a76f 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -29,3 +29,13 @@ jobs: - uses: actions-ext/yardang@main with: token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Wiki + run: | + pip install -e . + yardang wiki --output-dir docs/wiki + + - name: Upload Documentation to Wiki + uses: Andrew-Chen-Wang/github-wiki-action@v5 + with: + path: docs/wiki diff --git a/.gitignore b/.gitignore index 0df5c65..3aedeac 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,7 @@ docs/src/_build/ docs/api docs/index.md docs/html +docs/wiki docs/jupyter_execute index.md diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 5b1a431..44b5f00 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -677,3 +677,170 @@ Then in your documentation files, you can use sphinx-js directives: \`\`\`{js:automodule} myModule \`\`\` ```` + +## GitHub Wiki Integration + +Yardang can generate GitHub Wiki compatible markdown documentation using [sphinx-markdown-builder](https://github.com/liran-funaro/sphinx-markdown-builder). This allows you to publish your documentation to a GitHub Wiki in addition to (or instead of) a static HTML site. + +To generate wiki output, use the `yardang wiki` command instead of `yardang build`. + +Wiki output is configured in the `[tool.yardang.wiki]` section: + +```toml +[tool.yardang.wiki] +enabled = true +output-dir = "docs/wiki" +generate-sidebar = true +generate-footer = true +fix-links = true +footer-docs-url = "https://your-project.dev" +footer-repo-url = "https://github.com/your-org/your-project" +markdown-flavor = "github" +``` + +### `enabled` + +Enable the markdown builder extension. Must be `true` to use `yardang wiki`. Defaults to `false`. + +```toml +[tool.yardang.wiki] +enabled = true +``` + +### `output-dir` + +Output directory for the generated markdown files. Defaults to `"docs/wiki"`. + +```toml +[tool.yardang.wiki] +output-dir = "docs/wiki" +``` + +### `generate-sidebar` + +Generate a `_Sidebar.md` file for wiki navigation. Defaults to `true`. + +```toml +[tool.yardang.wiki] +generate-sidebar = true +``` + +### `generate-footer` + +Generate a `_Footer.md` file with links to docs and repo. Defaults to `true`. + +```toml +[tool.yardang.wiki] +generate-footer = true +``` + +### `fix-links` + +Fix internal markdown links for GitHub Wiki compatibility. Defaults to `true`. + +```toml +[tool.yardang.wiki] +fix-links = true +``` + +### `footer-docs-url` + +URL to the full documentation site (for the footer). + +```toml +[tool.yardang.wiki] +footer-docs-url = "https://your-project.dev" +``` + +### `footer-repo-url` + +URL to the repository (for the footer). + +```toml +[tool.yardang.wiki] +footer-repo-url = "https://github.com/your-org/your-project" +``` + +### `markdown-flavor` + +Markdown flavor to use. Set to `"github"` for GitHub-flavored markdown. Defaults to `"github"`. + +```toml +[tool.yardang.wiki] +markdown-flavor = "github" +``` + +### `markdown-anchor-sections` + +Add anchors before each section. Defaults to `true`. + +```toml +[tool.yardang.wiki] +markdown-anchor-sections = true +``` + +### `markdown-anchor-signatures` + +Add anchors before each function/class signature. Defaults to `true`. + +```toml +[tool.yardang.wiki] +markdown-anchor-signatures = true +``` + +### `markdown-bullet` + +Bullet character to use for lists. Defaults to `"-"`. + +```toml +[tool.yardang.wiki] +markdown-bullet = "-" +``` + +### Complete Example + +```toml +[tool.yardang] +title = "My Project" +root = "docs/index.md" +pages = ["docs/overview.md", "docs/api.md"] + +[tool.yardang.wiki] +enabled = true +output-dir = "docs/wiki" +generate-sidebar = true +generate-footer = true +footer-docs-url = "https://myproject.dev" +footer-repo-url = "https://github.com/myorg/myproject" +markdown-flavor = "github" +``` + +### Usage + +Generate GitHub Wiki output: + +```bash +yardang wiki +``` + +The output will be in the `docs/wiki/` directory (or the configured output directory). To publish to your GitHub Wiki: + +```bash +# Clone your wiki repository +git clone https://github.com/YOUR-ORG/YOUR-REPO.wiki.git + +# Copy the generated markdown files +cp -r docs/wiki/* YOUR-REPO.wiki/ + +# Commit and push +cd YOUR-REPO.wiki +git add . +git commit -m "Update wiki documentation" +git push +``` + +The generated wiki includes: +- `Home.md` - The main landing page (converted from index.md) +- `_Sidebar.md` - Navigation sidebar with links to all pages +- `_Footer.md` - Footer with links to documentation and repository +- All documentation pages converted to GitHub-flavored markdown diff --git a/pyproject.toml b/pyproject.toml index 06c4a5c..19fb280 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "breathe>=4.35.0", "sphinx-rust", "sphinx-js>=5.0.0", + "sphinx-markdown-builder>=0.6.9", ] [project.optional-dependencies] @@ -204,3 +205,12 @@ doc-formats = { calculator = "myst-nb" } [tool.yardang.sphinx-js] js-language = "javascript" js-source-path = "examples/js/src" + +[tool.yardang.wiki] +enabled = true +output-dir = "docs/wiki" +generate-sidebar = true +generate-footer = true +footer-docs-url = "https://yardang.python-templates.dev" +footer-repo-url = "https://github.com/python-project-templates/yardang" +markdown-flavor = "github" diff --git a/yardang/__init__.py b/yardang/__init__.py index dd1a0ef..318aa8d 100644 --- a/yardang/__init__.py +++ b/yardang/__init__.py @@ -1,5 +1,11 @@ __version__ = "0.4.1" -from .build import generate_docs_configuration, run_doxygen_if_needed +from .build import generate_docs_configuration, generate_wiki_configuration, run_doxygen_if_needed +from .wiki import process_wiki_output -__all__ = ("generate_docs_configuration", "run_doxygen_if_needed") +__all__ = ( + "generate_docs_configuration", + "generate_wiki_configuration", + "run_doxygen_if_needed", + "process_wiki_output", +) diff --git a/yardang/build.py b/yardang/build.py index c4a7b53..77fa4a1 100644 --- a/yardang/build.py +++ b/yardang/build.py @@ -10,7 +10,7 @@ from .utils import get_config -__all__ = ("generate_docs_configuration", "run_doxygen_if_needed") +__all__ = ("generate_docs_configuration", "run_doxygen_if_needed", "generate_wiki_configuration") def run_doxygen_if_needed( @@ -448,6 +448,39 @@ def customize(args): if js_args["root_for_relative_js_paths"]: js_args["root_for_relative_js_paths"] = str(Path(js_args["root_for_relative_js_paths"]).resolve()) + # Load wiki configuration from tool.yardang.wiki + wiki_config_base = f"{config_base}.wiki" + wiki_args = {} + for config_option, default in { + # wiki/markdown builder + "wiki_enabled": False, + "wiki_output_dir": "docs/wiki", + "wiki_generate_sidebar": True, + "wiki_generate_footer": True, + "wiki_fix_links": True, + "wiki_footer_docs_url": "", + "wiki_footer_repo_url": "", + # sphinx-markdown-builder options + "markdown_anchor_sections": True, + "markdown_anchor_signatures": True, + "markdown_docinfo": False, + "markdown_http_base": "", + "markdown_uri_doc_suffix": ".md", + "markdown_bullet": "-", + "markdown_flavor": "github", + }.items(): + # config keys in toml use hyphens, not underscores, and no wiki_ prefix for wiki-specific options + if config_option.startswith("wiki_"): + toml_key = config_option.replace("wiki_", "").replace("_", "-") + else: + toml_key = config_option.replace("_", "-") + wiki_args[config_option] = get_config(section=toml_key, base=wiki_config_base) + if wiki_args[config_option] is None: + wiki_args[config_option] = default + + # Determine if wiki/markdown output should be generated + use_wiki = wiki_args["wiki_enabled"] + # create a temporary directory to store the conf.py file in with TemporaryDirectory() as td: templateEnv = Environment(loader=FileSystemLoader(searchpath=str(Path(__file__).parent.resolve()))) @@ -472,9 +505,11 @@ def customize(args): use_breathe=use_breathe, use_sphinx_rust=use_sphinx_rust, use_sphinx_js=use_sphinx_js, + use_wiki=use_wiki, **breathe_args, **rust_args, **js_args, + **wiki_args, **configuration_args, ) @@ -520,3 +555,115 @@ def customize(args): fp.write("index.md\n") # yield folder path to sphinx build yield td + + +@contextmanager +def generate_wiki_configuration( + *, + project: str = "", + title: str = "", + module: str = "", + description: str = "", + author: str = "", + copyright: str = "", + version: str = "", + theme: str = "furo", + docs_root: str = "", + root: str = "", + cname: str = "", + pages: Optional[List] = None, + use_autoapi: Optional[bool] = None, + autoapi_ignore: Optional[List] = None, + custom_css: Optional[Path] = None, + custom_js: Optional[Path] = None, + config_base: str = "tool.yardang", + previous_versions: Optional[bool] = False, + adjust_arguments: Callable = None, + adjust_template: Callable = None, +): + """Generate Sphinx configuration for GitHub Wiki markdown output. + + A context manager similar to generate_docs_configuration, but configured + for building markdown output suitable for GitHub Wiki using + sphinx-markdown-builder. + + This adds the sphinx_markdown_builder extension and sets appropriate + options for GitHub-flavored markdown output. + + Args: + project: Project name. Falls back to ``[project].name`` or directory name. + title: Documentation title. Falls back to ``[tool.yardang].title`` or project name. + module: Python module name for autoapi. Falls back to project name with + hyphens replaced by underscores. + description: Project description for metadata. + author: Author name. Falls back to first entry in ``[project].authors``. + copyright: Copyright string. Falls back to author name. + version: Version string. Falls back to ``[project].version``. + theme: Sphinx theme name. Defaults to ``"furo"``. + docs_root: Base URL for hosted documentation. Used for canonical URLs. + root: Path to README or index file to use as documentation root. + cname: Custom domain name for GitHub Pages CNAME file. + pages: List of page paths to include in the toctree. + use_autoapi: Whether to use sphinx-autoapi for Python API docs. + custom_css: Path to custom CSS file. + custom_js: Path to custom JavaScript file. + config_base: Base key in pyproject.toml for configuration. + previous_versions: Whether to generate previous versions documentation. + adjust_arguments: Callback to modify template arguments before rendering. + adjust_template: Callback to modify the Jinja2 template before rendering. + + Yields: + tuple: (config_dir, wiki_args) where config_dir is the path to the directory + containing the generated conf.py file, and wiki_args is a dict with + wiki configuration for post-processing. + """ + + def add_markdown_builder(args): + # Enable wiki mode + args["use_wiki"] = True + # Call original adjust_arguments if provided + if adjust_arguments: + args = adjust_arguments(args) + return args + + with generate_docs_configuration( + project=project, + title=title, + module=module, + description=description, + author=author, + copyright=copyright, + version=version, + theme=theme, + docs_root=docs_root, + root=root, + cname=cname, + pages=pages, + use_autoapi=use_autoapi, + autoapi_ignore=autoapi_ignore, + custom_css=custom_css, + custom_js=custom_js, + config_base=config_base, + previous_versions=previous_versions, + adjust_arguments=add_markdown_builder, + adjust_template=adjust_template, + ) as config_dir: + # Read wiki args from config + wiki_config_base = f"{config_base}.wiki" + wiki_args = { + "wiki_output_dir": get_config(section="output-dir", base=wiki_config_base) or "docs/wiki", + "wiki_generate_sidebar": get_config(section="generate-sidebar", base=wiki_config_base), + "wiki_generate_footer": get_config(section="generate-footer", base=wiki_config_base), + "wiki_fix_links": get_config(section="fix-links", base=wiki_config_base), + "wiki_footer_docs_url": get_config(section="footer-docs-url", base=wiki_config_base) or "", + "wiki_footer_repo_url": get_config(section="footer-repo-url", base=wiki_config_base) or "", + } + # Apply defaults + if wiki_args["wiki_generate_sidebar"] is None: + wiki_args["wiki_generate_sidebar"] = True + if wiki_args["wiki_generate_footer"] is None: + wiki_args["wiki_generate_footer"] = True + if wiki_args["wiki_fix_links"] is None: + wiki_args["wiki_fix_links"] = True + + yield config_dir, wiki_args diff --git a/yardang/cli.py b/yardang/cli.py index 4fd95f0..09d8fee 100644 --- a/yardang/cli.py +++ b/yardang/cli.py @@ -6,7 +6,9 @@ from typer import Exit, Typer -from .build import generate_docs_configuration +from .build import generate_docs_configuration, generate_wiki_configuration +from .utils import get_config +from .wiki import process_wiki_output def build( @@ -81,10 +83,128 @@ def debug(): build(quiet=False, debug=True) +def wiki( + *, + quiet: bool = False, + debug: bool = False, + pdb: bool = False, + project: str = "", + title: str = "", + module: str = "", + description: str = "", + author: str = "", + copyright: str = "", + version: str = "", + theme: str = "furo", + docs_root: str = "", + root: str = "", + cname: str = "", + pages: Optional[List[Path]] = None, + use_autoapi: Optional[bool] = None, + custom_css: Optional[Path] = None, + custom_js: Optional[Path] = None, + config_base: Optional[str] = "tool.yardang", + previous_versions: Optional[bool] = False, + output_dir: Optional[str] = None, + skip_postprocess: bool = False, +): + """Generate GitHub Wiki compatible markdown documentation. + + Builds markdown output using sphinx-markdown-builder and post-processes + it to be compatible with GitHub Wiki format, including: + - Flattening directory structure + - Renaming index.md to Home.md + - Generating _Sidebar.md navigation + - Generating _Footer.md + - Fixing internal links + """ + # Get project name for wiki sidebar + project_name = project or get_config(section="name", base="project") or Path.cwd().name + + # Get pages from config if not provided + pages_list = pages or get_config(section="pages", base=config_base) or [] + if isinstance(pages_list, list): + pages_list = [str(p) for p in pages_list] + + with generate_wiki_configuration( + project=project, + title=title, + module=module, + description=description, + author=author, + copyright=copyright, + version=version, + theme=theme, + docs_root=docs_root, + root=root, + cname=cname, + pages=pages, + use_autoapi=use_autoapi, + custom_css=custom_css, + custom_js=custom_js, + config_base=config_base, + previous_versions=previous_versions, + ) as (config_dir, wiki_args): + # Determine output directory + wiki_output_dir = output_dir or wiki_args.get("wiki_output_dir", "docs/wiki") + + # Build markdown using sphinx-markdown-builder + build_cmd = [ + executable, + "-m", + "sphinx", + "-b", + "markdown", + ".", + wiki_output_dir, + "-c", + config_dir, + ] + + if debug: + print(" ".join(build_cmd)) + if quiet: + process = Popen(build_cmd) + else: + process = Popen(build_cmd, stderr=stderr, stdout=stdout) + while process.poll() is None: + sleep(0.1) + if process.returncode != 0: + if pdb: + import pdb + + pdb.set_trace() + raise Exit(process.returncode) + + # Post-process for GitHub Wiki compatibility + if not skip_postprocess: + if not quiet: + print("\nPost-processing markdown for GitHub Wiki...") + + process_wiki_output( + output_dir=Path(wiki_output_dir), + pages=pages_list, + project_name=project_name, + docs_url=wiki_args.get("wiki_footer_docs_url", ""), + repo_url=wiki_args.get("wiki_footer_repo_url", ""), + generate_sidebar_file=wiki_args.get("wiki_generate_sidebar", True), + generate_footer_file=wiki_args.get("wiki_generate_footer", True), + fix_links=wiki_args.get("wiki_fix_links", True), + ) + + if not quiet: + print(f"GitHub Wiki output generated in: {wiki_output_dir}") + print("\nTo use with GitHub Wiki:") + print(" 1. Clone your wiki: git clone https://github.com/YOUR/REPO.wiki.git") + print(f" 2. Copy contents of {wiki_output_dir}/ to the wiki repo") + print(" 3. Commit and push to publish") + + def main(): app = Typer() app.command("build")(build) app.command("debug")(debug) + app.command("wiki")(wiki) app() diff --git a/yardang/conf.py.j2 b/yardang/conf.py.j2 index 86c8b0a..c1d5363 100644 --- a/yardang/conf.py.j2 +++ b/yardang/conf.py.j2 @@ -44,6 +44,7 @@ use_autoapi = {{use_autoapi}} # noqa: F821 use_breathe = {{use_breathe}} # noqa: F821 use_sphinx_rust = {{use_sphinx_rust}} # noqa: F821 use_sphinx_js = {{use_sphinx_js}} # noqa: F821 +use_wiki = {{use_wiki}} # noqa: F821 ############################################################################################################# # _____ _ _ _ __ _ _ # @@ -84,6 +85,10 @@ if use_sphinx_rust: if use_sphinx_js: extensions.append("sphinx_js") +# Add sphinx-markdown-builder extension if wiki mode is enabled +if use_wiki: + extensions.append("sphinx_markdown_builder") + if use_autoapi in (True, None): # add if it is set to true or if it is set to None # NOTE: bug in autoapi that requires @@ -189,6 +194,18 @@ if use_sphinx_js: {% endif %} ts_type_bold = {{ts_type_bold}} +# sphinx-markdown-builder configuration for GitHub Wiki +if use_wiki: + markdown_anchor_sections = {{markdown_anchor_sections}} + markdown_anchor_signatures = {{markdown_anchor_signatures}} + markdown_docinfo = {{markdown_docinfo}} + {% if markdown_http_base %} + markdown_http_base = "{{markdown_http_base}}" + {% endif %} + markdown_uri_doc_suffix = "{{markdown_uri_doc_suffix}}" + markdown_bullet = "{{markdown_bullet}}" + markdown_flavor = "{{markdown_flavor}}" + # autosummary autosummary_generate = True # if using autosummary, autogenerate diff --git a/yardang/tests/test_wiki.py b/yardang/tests/test_wiki.py new file mode 100644 index 0000000..fd97eed --- /dev/null +++ b/yardang/tests/test_wiki.py @@ -0,0 +1,317 @@ +"""Tests for GitHub Wiki generation in yardang.""" + +import os +from pathlib import Path + + +class TestWikiConfiguration: + """Tests for wiki configuration loading and generation.""" + + def test_wiki_config_loading_from_pyproject(self, tmp_path): + """Test that wiki configuration is loaded from pyproject.toml.""" + pyproject_content = """ +[project] +name = "test-project" +version = "1.0.0" + +[tool.yardang] +title = "Test Project" +root = "README.md" +use-autoapi = false + +[tool.yardang.wiki] +enabled = true +output-dir = "docs/wiki" +generate-sidebar = true +generate-footer = true +fix-links = true +footer-docs-url = "https://example.com/docs" +footer-repo-url = "https://github.com/example/repo" +markdown-flavor = "github" +""" + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text(pyproject_content) + + readme_path = tmp_path / "README.md" + readme_path.write_text("# Test Project\n\nTest content.") + + original_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + from yardang.utils import get_config + + # Test that wiki config is loaded correctly + wiki_enabled = get_config(section="enabled", base="tool.yardang.wiki") + assert wiki_enabled is True + + wiki_output_dir = get_config(section="output-dir", base="tool.yardang.wiki") + assert wiki_output_dir == "docs/wiki" + + generate_sidebar = get_config(section="generate-sidebar", base="tool.yardang.wiki") + assert generate_sidebar is True + + markdown_flavor = get_config(section="markdown-flavor", base="tool.yardang.wiki") + assert markdown_flavor == "github" + + footer_docs_url = get_config(section="footer-docs-url", base="tool.yardang.wiki") + assert footer_docs_url == "https://example.com/docs" + finally: + os.chdir(original_cwd) + + def test_wiki_config_defaults(self, tmp_path): + """Test that wiki configuration has sensible defaults when not specified.""" + pyproject_content = """ +[project] +name = "test-project" +version = "1.0.0" + +[tool.yardang] +title = "Test Project" +root = "README.md" +""" + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text(pyproject_content) + + readme_path = tmp_path / "README.md" + readme_path.write_text("# Test Project\n\nTest content.") + + original_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + from yardang.utils import get_config + + # Test that missing wiki config returns None + wiki_enabled = get_config(section="enabled", base="tool.yardang.wiki") + assert wiki_enabled is None + + wiki_output_dir = get_config(section="output-dir", base="tool.yardang.wiki") + assert wiki_output_dir is None + finally: + os.chdir(original_cwd) + + def test_generate_docs_with_wiki_enabled(self, tmp_path): + """Test that generate_docs_configuration includes wiki/markdown builder settings.""" + pyproject_content = """ +[project] +name = "test-project" +version = "1.0.0" + +[tool.yardang] +title = "Test Project" +root = "README.md" +use-autoapi = false + +[tool.yardang.wiki] +enabled = true +markdown-flavor = "github" +markdown-anchor-sections = true +""" + pyproject_path = tmp_path / "pyproject.toml" + pyproject_path.write_text(pyproject_content) + + readme_path = tmp_path / "README.md" + readme_path.write_text("# Test Project\n\nTest content.") + + original_cwd = os.getcwd() + try: + os.chdir(tmp_path) + + from yardang.build import generate_docs_configuration + + with generate_docs_configuration() as conf_dir: + conf_path = Path(conf_dir) / "conf.py" + conf_content = conf_path.read_text() + + # Check that use_wiki is set + assert "use_wiki = True" in conf_content + + # Check that markdown builder config is present when wiki is enabled + assert "markdown_flavor" in conf_content + assert "github" in conf_content + finally: + os.chdir(original_cwd) + + +class TestWikiPostProcessor: + """Tests for wiki post-processing functions.""" + + def test_get_page_title_from_h1(self, tmp_path): + """Test extracting title from H1 heading.""" + from yardang.wiki import get_page_title + + md_file = tmp_path / "test.md" + md_file.write_text("# My Page Title\n\nSome content here.") + + title = get_page_title(md_file) + assert title == "My Page Title" + + def test_get_page_title_fallback_to_filename(self, tmp_path): + """Test fallback to filename when no H1 heading exists.""" + from yardang.wiki import get_page_title + + md_file = tmp_path / "my-page-name.md" + md_file.write_text("Some content without heading.") + + title = get_page_title(md_file) + assert title == "My Page Name" + + def test_get_page_title_index_becomes_home(self, tmp_path): + """Test that index.md gets 'Home' as title.""" + from yardang.wiki import get_page_title + + md_file = tmp_path / "index.md" + md_file.write_text("Some content without heading.") + + title = get_page_title(md_file) + assert title == "Home" + + def test_convert_filename_to_wiki_format(self): + """Test filename conversion for wiki format.""" + from yardang.wiki import convert_filename_to_wiki_format + + assert convert_filename_to_wiki_format("overview.md") == "overview" + assert convert_filename_to_wiki_format("index.md") == "Home" + assert convert_filename_to_wiki_format("readme.md") == "Home" + assert convert_filename_to_wiki_format("docs/src/api.md") == "api" + + def test_fix_wiki_links_basic(self): + """Test fixing internal markdown links.""" + from yardang.wiki import fix_wiki_links + + content = "Check [the overview](overview.md) for more info." + page_map = {"overview.md": "overview"} + + fixed = fix_wiki_links(content, page_map) + assert "[the overview](overview)" in fixed + + def test_fix_wiki_links_with_anchors(self): + """Test fixing links with anchors.""" + from yardang.wiki import fix_wiki_links + + content = "See [installation](installation.md#quick-start) guide." + page_map = {"installation.md": "installation"} + + fixed = fix_wiki_links(content, page_map) + assert "[installation](installation#quick-start)" in fixed + + def test_fix_wiki_links_preserves_external_links(self): + """Test that external links are preserved.""" + from yardang.wiki import fix_wiki_links + + content = "Visit [GitHub](https://github.com) for more info." + page_map = {} + + fixed = fix_wiki_links(content, page_map) + assert "[GitHub](https://github.com)" in fixed + + def test_generate_sidebar(self, tmp_path): + """Test sidebar generation.""" + from yardang.wiki import generate_sidebar + + # Create some test files + (tmp_path / "Home.md").write_text("# Welcome\n\nHome page content.") + (tmp_path / "overview.md").write_text("# Overview\n\nOverview content.") + (tmp_path / "installation.md").write_text("# Installation\n\nInstall steps.") + + pages = ["docs/src/overview.md", "docs/src/installation.md"] + sidebar = generate_sidebar(tmp_path, pages, project_name="My Project") + + assert "### My Project" in sidebar + assert "[Home](Home)" in sidebar + assert "[Overview](overview)" in sidebar + assert "[Installation](installation)" in sidebar + + # Check file was created + sidebar_file = tmp_path / "_Sidebar.md" + assert sidebar_file.exists() + + def test_generate_footer(self, tmp_path): + """Test footer generation.""" + from yardang.wiki import generate_footer + + footer = generate_footer( + tmp_path, + project_name="My Project", + docs_url="https://myproject.dev", + repo_url="https://github.com/myorg/myproject", + ) + + assert "[📚 Full Documentation](https://myproject.dev)" in footer + assert "[💻 Repository](https://github.com/myorg/myproject)" in footer + + # Check file was created + footer_file = tmp_path / "_Footer.md" + assert footer_file.exists() + + def test_rename_index_to_home(self, tmp_path): + """Test renaming index.md to Home.md.""" + from yardang.wiki import rename_index_to_home + + # Create index.md + index_file = tmp_path / "index.md" + index_file.write_text("# Welcome\n\nMain page content.") + + rename_index_to_home(tmp_path) + + # Check Home.md exists and index.md doesn't + assert not index_file.exists() + assert (tmp_path / "Home.md").exists() + + def test_flatten_directory_structure(self, tmp_path): + """Test flattening nested directories.""" + from yardang.wiki import flatten_directory_structure + + # Create nested structure + subdir = tmp_path / "subdir" + subdir.mkdir() + (subdir / "page.md").write_text("# Page\n\nContent.") + + nested = tmp_path / "docs" / "api" + nested.mkdir(parents=True) + (nested / "reference.md").write_text("# API Reference\n\nAPI content.") + + flatten_directory_structure(tmp_path) + + # Check files are flattened + assert (tmp_path / "subdir-page.md").exists() + assert (tmp_path / "docs-api-reference.md").exists() + + # Check nested dirs are removed + assert not (tmp_path / "subdir" / "page.md").exists() + + def test_process_wiki_output_full(self, tmp_path): + """Test full wiki output processing.""" + from yardang.wiki import process_wiki_output + + # Create test markdown output structure + (tmp_path / "index.md").write_text("# Welcome\n\nSee [overview](overview.md).") + (tmp_path / "overview.md").write_text("# Overview\n\nOverview content.") + (tmp_path / "api.md").write_text("# API\n\nAPI docs.") + + process_wiki_output( + output_dir=tmp_path, + pages=["overview.md", "api.md"], + project_name="Test Project", + docs_url="https://test.dev", + repo_url="https://github.com/test/project", + ) + + # Check Home.md exists (renamed from index.md) + assert (tmp_path / "Home.md").exists() + assert not (tmp_path / "index.md").exists() + + # Check sidebar was generated + assert (tmp_path / "_Sidebar.md").exists() + sidebar = (tmp_path / "_Sidebar.md").read_text() + assert "Test Project" in sidebar + + # Check footer was generated + assert (tmp_path / "_Footer.md").exists() + footer = (tmp_path / "_Footer.md").read_text() + assert "test.dev" in footer + + # Check links were fixed in Home.md + home_content = (tmp_path / "Home.md").read_text() + assert "(overview)" in home_content diff --git a/yardang/wiki.py b/yardang/wiki.py new file mode 100644 index 0000000..e4f8a63 --- /dev/null +++ b/yardang/wiki.py @@ -0,0 +1,447 @@ +"""GitHub Wiki markdown generation and post-processing. + +This module provides functionality to generate GitHub Wiki compatible markdown +from Sphinx documentation using sphinx-markdown-builder, with post-processing +to create proper sidebar navigation and fix internal links. +""" + +import re +import shutil +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +__all__ = ( + "generate_sidebar", + "generate_footer", + "fix_wiki_links", + "process_wiki_output", + "get_page_title", +) + + +def get_page_title(filepath: Path) -> str: + """Extract the title from a markdown file. + + Looks for the first H1 heading (# Title) in the file. + Falls back to the filename without extension. + + Args: + filepath: Path to the markdown file. + + Returns: + The extracted title or filename-based title. + """ + try: + content = filepath.read_text(encoding="utf-8") + # Look for first H1 heading + match = re.search(r"^#\s+(.+?)(?:\s*\{.*\})?$", content, re.MULTILINE) + if match: + title = match.group(1).strip() + # Remove any remaining markdown formatting + title = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", title) + title = re.sub(r"`([^`]+)`", r"\1", title) + return title + except Exception: + pass + + # Fallback to filename + name = filepath.stem + # Convert Home or index to Home + if name.lower() in ("index", "readme"): + return "Home" + # Convert kebab-case or snake_case to Title Case + name = name.replace("-", " ").replace("_", " ") + return name.title() + + +def convert_filename_to_wiki_format(filename: str) -> str: + """Convert a filename to GitHub Wiki format. + + GitHub Wiki uses spaces in URLs which map to filenames with hyphens or spaces. + We'll use the standard approach of keeping the filename but fixing the extension. + + Args: + filename: Original filename (may include path components). + + Returns: + Wiki-compatible filename. + """ + # Remove path components for wiki (wiki is flat) + name = Path(filename).stem + # Handle index/readme -> Home + if name.lower() in ("index", "readme"): + return "Home" + return name + + +def fix_wiki_links(content: str, all_pages: Dict[str, str]) -> str: + """Fix internal links to use GitHub Wiki format. + + Converts relative markdown links to GitHub Wiki internal links. + GitHub Wiki links use [[Page Name]] or [text](Page-Name) format. + + Args: + content: Markdown content to process. + all_pages: Dict mapping original paths to wiki page names. + + Returns: + Content with fixed links. + """ + + # Fix standard markdown links [text](path) + def replace_link(match): + text = match.group(1) + path = match.group(2) + + # Skip external links + if path.startswith(("http://", "https://", "mailto:", "#")): + return match.group(0) + + # Handle anchor links within same page + if path.startswith("#"): + return match.group(0) + + # Extract path and anchor + anchor = "" + if "#" in path: + path, anchor = path.split("#", 1) + anchor = "#" + anchor + + # Remove .md extension if present + if path.endswith(".md"): + path = path[:-3] + + # Convert path to wiki page name + page_name = convert_filename_to_wiki_format(path) + + # Check if it's a known page + for orig_path, wiki_name in all_pages.items(): + if path in orig_path or orig_path.endswith(path): + page_name = wiki_name + break + + return f"[{text}]({page_name}{anchor})" + + content = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", replace_link, content) + + return content + + +def extract_toctree_entries(content: str) -> List[Tuple[str, str]]: + """Extract toctree entries from markdown content. + + Parses MyST-style toctree directives to find linked pages. + + Args: + content: Markdown content with toctree directives. + + Returns: + List of (title, path) tuples for toctree entries. + """ + entries = [] + + # Match MyST toctree blocks + toctree_pattern = r"```\{toctree\}.*?```" + matches = re.findall(toctree_pattern, content, re.DOTALL) + + for toctree_block in matches: + # Extract entries from toctree block + lines = toctree_block.split("\n") + in_content = False + for line in lines: + line = line.strip() + # Skip directive header and options + if line.startswith("```") or line.startswith(":") or line.startswith("---"): + if line == "---": + in_content = True + continue + if not in_content and line.startswith(":"): + continue + if line.startswith("---"): + in_content = True + continue + if in_content and line and not line.startswith(":"): + # This is an entry + path = line.strip() + if path: + entries.append((None, path)) + + return entries + + +def generate_sidebar( + output_dir: Path, + pages: List[str], + project_name: str = "", + *, + include_home: bool = True, +) -> str: + """Generate a _Sidebar.md file for GitHub Wiki. + + Creates a sidebar navigation file based on the documentation structure. + + Args: + output_dir: Directory containing the markdown output. + pages: List of page paths from the yardang configuration. + project_name: Project name for the sidebar header. + include_home: Whether to include a Home link at the top. + + Returns: + The generated sidebar content. + """ + lines = [] + + if project_name: + lines.append(f"### {project_name}") + lines.append("") + + if include_home: + lines.append("* [Home](Home)") + + # Build navigation from pages + for page_path in pages: + # Get the wiki filename + wiki_name = convert_filename_to_wiki_format(page_path) + + # Try to extract title from the file + md_file = output_dir / f"{wiki_name}.md" + if md_file.exists(): + title = get_page_title(md_file) + else: + title = wiki_name.replace("-", " ").replace("_", " ").title() + + lines.append(f"* [{title}]({wiki_name})") + + # Check for additional pages not in the explicit list + for md_file in sorted(output_dir.glob("*.md")): + if md_file.name.startswith("_"): + continue + wiki_name = md_file.stem + if wiki_name.lower() == "home": + continue + + # Check if already included + already_included = False + for page_path in pages: + if convert_filename_to_wiki_format(page_path) == wiki_name: + already_included = True + break + + if not already_included: + title = get_page_title(md_file) + lines.append(f"* [{title}]({wiki_name})") + + content = "\n".join(lines) + + # Write the sidebar file + sidebar_file = output_dir / "_Sidebar.md" + sidebar_file.write_text(content, encoding="utf-8") + + return content + + +def generate_footer( + output_dir: Path, + project_name: str = "", + docs_url: str = "", + repo_url: str = "", +) -> str: + """Generate a _Footer.md file for GitHub Wiki. + + Creates a footer with links to the main documentation and repository. + + Args: + output_dir: Directory containing the markdown output. + project_name: Project name. + docs_url: URL to the main documentation site. + repo_url: URL to the repository. + + Returns: + The generated footer content. + """ + lines = ["---", ""] + + links = [] + if docs_url: + links.append(f"[📚 Full Documentation]({docs_url})") + if repo_url: + links.append(f"[💻 Repository]({repo_url})") + + if links: + lines.append(" | ".join(links)) + else: + lines.append(f"*Generated from {project_name} documentation*") + + content = "\n".join(lines) + + # Write the footer file + footer_file = output_dir / "_Footer.md" + footer_file.write_text(content, encoding="utf-8") + + return content + + +def rename_index_to_home(output_dir: Path) -> None: + """Rename index.md to Home.md for GitHub Wiki. + + GitHub Wiki uses Home.md as the landing page. + + Args: + output_dir: Directory containing the markdown output. + """ + index_file = output_dir / "index.md" + home_file = output_dir / "Home.md" + + if index_file.exists() and not home_file.exists(): + shutil.move(str(index_file), str(home_file)) + + +def flatten_directory_structure(output_dir: Path, max_filename_length: int = 200) -> Dict[str, str]: + """Flatten nested directory structure for GitHub Wiki. + + GitHub Wiki doesn't support nested directories, so we need to + flatten the structure and rename files appropriately. + + Args: + output_dir: Directory containing the markdown output. + max_filename_length: Maximum length for generated filenames. + + Returns: + Dict mapping original paths to new wiki page names. + """ + page_map = {} + + # Patterns to skip (these are build artifacts, not documentation) + skip_patterns = ["jupyter_execute", "_build", ".ipynb_checkpoints", "__pycache__"] + + # Find all markdown files recursively + for md_file in output_dir.rglob("*.md"): + # Skip files in directories matching skip patterns + rel_path = md_file.relative_to(output_dir) + should_skip = any(pattern in str(rel_path) for pattern in skip_patterns) + if should_skip: + # Remove these files as they're build artifacts + try: + md_file.unlink() + except OSError: + pass + continue + + if md_file.parent == output_dir: + # Already at top level + wiki_name = md_file.stem + if wiki_name.lower() in ("index", "readme"): + wiki_name = "Home" + page_map[str(rel_path)] = wiki_name + continue + + # Build a flattened name from the path + parts = list(rel_path.parts) + + # Remove 'index.md' at the end and use parent name + if parts[-1].lower() in ("index.md", "readme.md"): + parts = parts[:-1] + if parts: + wiki_name = "-".join(parts) + else: + wiki_name = "Home" + else: + # Remove .md extension + parts[-1] = Path(parts[-1]).stem + wiki_name = "-".join(parts) + + # Truncate if name is too long (with room for .md extension) + if len(wiki_name) > max_filename_length: + # Use hash to make unique truncated name + import hashlib + + name_hash = hashlib.md5(wiki_name.encode()).hexdigest()[:8] + wiki_name = wiki_name[: max_filename_length - 10] + "-" + name_hash + + page_map[str(rel_path)] = wiki_name + + # Move file to top level with new name + new_path = output_dir / f"{wiki_name}.md" + if not new_path.exists(): + try: + shutil.move(str(md_file), str(new_path)) + except OSError: + # Skip files that can't be moved (e.g., name too long on some systems) + pass + + # Remove empty directories + for subdir in sorted(output_dir.rglob("*"), reverse=True): + if subdir.is_dir(): + try: + if not any(subdir.iterdir()): + subdir.rmdir() + except OSError: + pass + + return page_map + + +def process_wiki_output( + output_dir: Path, + pages: Optional[List[str]] = None, + project_name: str = "", + docs_url: str = "", + repo_url: str = "", + *, + generate_sidebar_file: bool = True, + generate_footer_file: bool = True, + fix_links: bool = True, +) -> None: + """Process sphinx-markdown-builder output for GitHub Wiki compatibility. + + This is the main entry point for wiki post-processing. It: + 1. Flattens the directory structure + 2. Renames index.md to Home.md + 3. Fixes internal links + 4. Generates _Sidebar.md + 5. Generates _Footer.md + + Args: + output_dir: Directory containing the markdown output from sphinx-markdown-builder. + pages: List of page paths from the yardang configuration. + project_name: Project name for sidebar/footer. + docs_url: URL to the main documentation site. + repo_url: URL to the repository. + generate_sidebar_file: Whether to generate _Sidebar.md. + generate_footer_file: Whether to generate _Footer.md. + fix_links: Whether to fix internal links. + """ + output_dir = Path(output_dir) + pages = pages or [] + + if not output_dir.exists(): + raise FileNotFoundError(f"Output directory not found: {output_dir}") + + # Flatten directory structure + page_map = flatten_directory_structure(output_dir) + + # Rename index to Home + rename_index_to_home(output_dir) + + # Update page map for Home + for orig_path, wiki_name in list(page_map.items()): + if wiki_name.lower() in ("index", "readme"): + page_map[orig_path] = "Home" + + # Fix links in all markdown files + if fix_links: + for md_file in output_dir.glob("*.md"): + if md_file.name.startswith("_"): + continue + + content = md_file.read_text(encoding="utf-8") + fixed_content = fix_wiki_links(content, page_map) + md_file.write_text(fixed_content, encoding="utf-8") + + # Generate sidebar + if generate_sidebar_file: + generate_sidebar(output_dir, pages, project_name) + + # Generate footer + if generate_footer_file: + generate_footer(output_dir, project_name, docs_url, repo_url)