diff --git a/MODULE.bazel b/MODULE.bazel index e090cf770df..4b4b47ed533 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -377,6 +377,49 @@ orfs.default( # --- External archives --- +# --- Pandoc pre-built binaries (used by //docs:man_pages) --- +# Pandoc is not on BCR; download platform-specific release archives. +# SHA-256 values come from the GitHub Releases API digest field. +_PANDOC_VERSION = "3.10" + +_PANDOC_BUILD_FILE = """\ +filegroup( + name = "pandoc", + srcs = glob(["**/bin/pandoc"]), + visibility = ["//visibility:public"], +) +""" + +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "pandoc_linux_amd64", + build_file_content = _PANDOC_BUILD_FILE, + sha256 = "e0f8af62d0f267d22baa5bcefe6d5dda3a097ccc60de794b759fe03159923244", + url = "https://github.com/jgm/pandoc/releases/download/{v}/pandoc-{v}-linux-amd64.tar.gz".format(v = _PANDOC_VERSION), +) + +http_archive( + name = "pandoc_linux_arm64", + build_file_content = _PANDOC_BUILD_FILE, + sha256 = "55413dfb0c1aec861641fe858f1f73e84848f3db497b1c0c02e62887ea76f4a4", + url = "https://github.com/jgm/pandoc/releases/download/{v}/pandoc-{v}-linux-arm64.tar.gz".format(v = _PANDOC_VERSION), +) + +http_archive( + name = "pandoc_macos_x86_64", + build_file_content = _PANDOC_BUILD_FILE, + sha256 = "6334f4d9af7c9e37e761dfad56fa5507685f6d29724ebf31c4be6d5c654a3161", + url = "https://github.com/jgm/pandoc/releases/download/{v}/pandoc-{v}-x86_64-macOS.zip".format(v = _PANDOC_VERSION), +) + +http_archive( + name = "pandoc_macos_arm64", + build_file_content = _PANDOC_BUILD_FILE, + sha256 = "d9cad01d96ae774a0dc8c8c45bb1ad3e4c5ff2cc2e24f45958f5f9b7974aee34", + url = "https://github.com/jgm/pandoc/releases/download/{v}/pandoc-{v}-arm64-macOS.zip".format(v = _PANDOC_VERSION), +) + # Alias @slang -> @sv-lang//:libsvlang so submodule BUILD files that # reference @slang resolve without modification. OpenROAD source uses # @sv-lang directly; this alias only exists for the slang-elab submodule. diff --git a/bazel/BUILD b/bazel/BUILD index 696a48a8dbb..83acc253256 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -3,11 +3,69 @@ load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") +load("@rules_python//python:defs.bzl", "py_binary") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") package(features = ["layering_check"]) +# --------------------------------------------------------------------------- +# Pandoc platform selection +# --------------------------------------------------------------------------- + +config_setting( + name = "_linux_x86_64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "_linux_aarch64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], +) + +config_setting( + name = "_macos_x86_64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], +) + +config_setting( + name = "_macos_aarch64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], +) + +# Pre-built pandoc binary, platform-selected at build time. +filegroup( + name = "pandoc", + srcs = select({ + ":_linux_aarch64": ["@pandoc_linux_arm64//:pandoc"], + ":_linux_x86_64": ["@pandoc_linux_amd64//:pandoc"], + ":_macos_aarch64": ["@pandoc_macos_arm64//:pandoc"], + ":_macos_x86_64": ["@pandoc_macos_x86_64//:pandoc"], + }), + visibility = ["//visibility:public"], +) + +# Driver script that orchestrates man page generation inside Bazel's sandbox. +py_binary( + name = "manpages_impl", + srcs = ["manpages_impl.py"], + visibility = ["//visibility:public"], +) + +# --------------------------------------------------------------------------- + exports_files( [ "bzl_fmt_test.sh", diff --git a/bazel/man_pages.bzl b/bazel/man_pages.bzl index b4240c87269..930e6c28098 100644 --- a/bazel/man_pages.bzl +++ b/bazel/man_pages.bzl @@ -1,57 +1,66 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026, The OpenROAD Authors -"""Bazel rule that invokes docs/Makefile to produce cat/ and html/ man pages. +"""Hermetic Bazel rule that generates cat/ and HTML/ man pages. -The output filenames aren't known at analysis time (they depend on which -modules exist under src/*/), so outputs are declared as TreeArtifacts and -the real work is delegated to the `bazel-manpages` Makefile target. +All tools (pandoc, Python interpreter) and all inputs (README files, +messages.txt files, man1 source markdown, Python scripts) are declared +as explicit Bazel dependencies. No host PATH look-ups, no make, no +nroff — pandoc --to=plain replaces nroff+col for cat pages. -Host requirements: pandoc, nroff (groff), col (bsdextrautils), python3>=3.10. +Output filenames are not known at analysis time (they depend on how +many Tcl commands each module exposes), so outputs are declared as +TreeArtifacts. """ -def _man_pages_resource_set(_os, _num_inputs): - # The 'cat web' make below fans out with -j$(nproc), so this action uses - # the whole host. Reserve all local CPUs to keep Bazel from co-scheduling - # other heavy actions alongside it and oversubscribing the machine. Bazel - # clamps the request to the cores actually available, so this is safe on - # small CI hosts too. - return {"cpu": 512.0} - def _man_pages_impl(ctx): cat_dir = ctx.actions.declare_directory("cat") html_dir = ctx.actions.declare_directory("html") - command = """ -set -euo pipefail -CAT_OUT="$PWD/{cat_out}" -HTML_OUT="$PWD/{html_out}" -# Two phases: 'preprocess' (serial) generates the md/man*/*.md sources, then -# 'cat web' fan out pandoc/nroff in parallel. They cannot share one -j make -# invocation: cat/web read the md files preprocess produces, and a parallel -# build has no dependency edge forcing preprocess to finish first. Running -# 'cat web' as a second invocation also re-parses the Makefile so its -# $(wildcard md/man*/*.md) picks up the freshly generated sources. -# nproc is GNU coreutils (absent on stock macOS); fall back to sysctl, then 4. -JOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" -make --no-print-directory -C docs -f Makefile preprocess \\ - CAT_ROOT_DIR="$CAT_OUT" HTML_ROOT_DIR="$HTML_OUT" -make --no-print-directory -j"$JOBS" -C docs -f Makefile cat web \\ - CAT_ROOT_DIR="$CAT_OUT" HTML_ROOT_DIR="$HTML_OUT" -""".format( - cat_out = cat_dir.path, - html_out = html_dir.path, + pandoc = ctx.file._pandoc + impl = ctx.executable._manpages_impl + + args = ctx.actions.args() + args.add("--pandoc", pandoc) + args.add("--cat-out", cat_dir.path) + args.add("--html-out", html_dir.path) + + for f in ctx.files.scripts: + if f.basename.endswith(".py"): + args.add("--script", f) + + for readme in ctx.files.readmes: + # Derive module name: src/ant/README.md → ant + parts = readme.short_path.split("/") + module = parts[-2] if len(parts) >= 2 else readme.basename + args.add("--readme", "{}:{}".format(module, readme.path)) + + for msg in ctx.files.messages: + # Derive module name: src/ant/messages.txt → ant + parts = msg.short_path.split("/") + module = parts[-2] if len(parts) >= 2 else msg.basename + args.add("--messages", "{}:{}".format(module, msg.path)) + + for f in ctx.files.docs_srcs: + if "/man1/" in f.path and f.basename.endswith(".md"): + args.add("--man1-src", f) + + all_inputs = depset( + ctx.files.docs_srcs + + ctx.files.scripts + + ctx.files.readmes + + ctx.files.messages + + [pandoc], + transitive = [ctx.attr._manpages_impl[DefaultInfo].default_runfiles.files], ) - ctx.actions.run_shell( - resource_set = _man_pages_resource_set, + ctx.actions.run( outputs = [cat_dir, html_dir], - inputs = ctx.files.docs_srcs + ctx.files.scripts + ctx.files.readmes + ctx.files.messages, - command = command, + inputs = all_inputs, + executable = impl, + arguments = [args], mnemonic = "ManPages", progress_message = "Generating man pages (cat + html)", - use_default_shell_env = True, - execution_requirements = {"no-sandbox": "1"}, ) return [DefaultInfo( @@ -63,11 +72,11 @@ man_pages = rule( implementation = _man_pages_impl, attrs = { "docs_srcs": attr.label_list( - doc = "All source files under docs/ needed by the Makefile.", + doc = "Source .md files under docs/md/ needed for man page generation.", allow_files = True, ), "messages": attr.label_list( - doc = "Module messages.txt files needed for man3 page generation.", + doc = "Module messages.txt files for man3 page generation.", allow_files = [".txt"], ), "readmes": attr.label_list( @@ -75,8 +84,18 @@ man_pages = rule( allow_files = [".md"], ), "scripts": attr.label_list( - doc = "Python/shell scripts for man page generation.", + doc = "Python scripts for man page generation.", allow_files = True, ), + "_manpages_impl": attr.label( + default = "//bazel:manpages_impl", + executable = True, + cfg = "exec", + ), + "_pandoc": attr.label( + default = "//bazel:pandoc", + allow_single_file = True, + cfg = "exec", + ), }, ) diff --git a/bazel/manpages_impl.py b/bazel/manpages_impl.py new file mode 100644 index 00000000000..14936ff4a0a --- /dev/null +++ b/bazel/manpages_impl.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026, The OpenROAD Authors +"""Hermetic Bazel driver for OpenROAD man page generation. + +Replaces the docs/Makefile + make + nroff chain with an explicit +Python + pandoc workflow that Bazel can fully sandbox. All inputs +(pandoc binary, README files, messages, scripts) are passed as +explicit CLI arguments; nothing is looked up from PATH or the host +filesystem. + +Usage (generated by //bazel:man_pages.bzl): + python manpages_impl.py \\ + --pandoc /path/to/pandoc \\ + --script docs/src/scripts/md_roff_compat.py \\ + --script docs/src/scripts/manpage.py \\ + --script docs/src/scripts/extract_utils.py \\ + --readme ant:src/ant/README.md \\ + --readme cts:src/cts/README.md \\ + ... + --messages ant:src/ant/messages.txt \\ + ... + --man1-src docs/md/man1/openroad.md \\ + ... + --cat-out bazel-out/.../docs/cat \\ + --html-out bazel-out/.../docs/html +""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile + + +def _run_pandoc(pandoc, src, dst, to_format): + """Run pandoc to convert *src* to *dst* using *to_format* writer.""" + subprocess.run( + [pandoc, "-s", "-t", to_format, src, "-o", dst, "--quiet"], + check=True, + ) + + +def _process_section( + pandoc, md_dir, roff_scratch, html_dir, cat_dir, section_num, skip_stems=None +): + """Convert every .md in *md_dir* to roff, HTML, and cat pages. + + Files whose stem (filename without .md) is in *skip_stems* are + left alone — used to avoid re-converting module READMEs as if + they were per-function man pages. + """ + skip_stems = skip_stems or set() + for fname in sorted(os.listdir(md_dir)): + if not fname.endswith(".md"): + continue + stem = fname[:-3] + if stem in skip_stems: + continue + src = os.path.join(md_dir, fname) + roff = os.path.join(roff_scratch, f"{stem}.{section_num}") + _run_pandoc(pandoc, src, roff, "man") + _run_pandoc(pandoc, roff, os.path.join(html_dir, f"{stem}.html"), "html") + # pandoc plain replaces nroff -man | col -b + _run_pandoc( + pandoc, roff, os.path.join(cat_dir, f"{stem}.{section_num}"), "plain" + ) + + +def main(): + """Orchestrate man page generation from the Bazel sandbox. + + Mirrors the source tree layout that md_roff_compat.py expects, + runs generation scripts, then converts all resulting markdown + to roff / HTML / cat pages via pandoc. + """ + p = argparse.ArgumentParser() + p.add_argument("--pandoc", required=True, help="Path to pandoc binary") + p.add_argument( + "--script", + action="append", + default=[], + help="Python script path (may be repeated)", + ) + p.add_argument( + "--readme", + action="append", + default=[], + help="module:path pairs for README.md files", + ) + p.add_argument( + "--messages", + action="append", + default=[], + help="module:path pairs for messages.txt files", + ) + p.add_argument( + "--man1-src", action="append", default=[], help="man1 source .md files" + ) + p.add_argument("--cat-out", required=True, help="Output directory for cat pages") + p.add_argument("--html-out", required=True, help="Output directory for html pages") + args = p.parse_args() + + with tempfile.TemporaryDirectory() as workdir: + # Copy pandoc into a writable temp location and make it executable. + # Input files in the Bazel sandbox may be on a read-only mount, so + # chmod-ing the original path would fail on some platforms/zip archives. + pandoc = os.path.join(workdir, "pandoc") + shutil.copy(args.pandoc, pandoc) + os.chmod(pandoc, 0o755) + # Mirror the directory layout that md_roff_compat.py expects when + # run from docs/: + # ./md/man2/{module}.md ← README copies + # ../src/{module}/messages.txt + docs_dir = os.path.join(workdir, "docs") + man1_md = os.path.join(docs_dir, "md", "man1") + man2_md = os.path.join(docs_dir, "md", "man2") + man3_md = os.path.join(docs_dir, "md", "man3") + scripts_dir = os.path.join(docs_dir, "src", "scripts") + for d in [man1_md, man2_md, man3_md, scripts_dir]: + os.makedirs(d) + + # Copy Python helper scripts so their relative imports resolve. + for script_path in args.script: + if script_path.endswith(".py"): + shutil.copy(script_path, scripts_dir) + + # Copy man1 source files. + for f in args.man1_src: + shutil.copy(f, man1_md) + + # Copy README files as {module}.md (replacing link_readmes.sh). + for spec in args.readme: + module, path = spec.split(":", 1) + shutil.copy(path, os.path.join(man2_md, f"{module}.md")) + + # Copy each module's messages.txt to ../src/{module}/messages.txt + # so md_roff_compat.py finds them relative to the docs/ directory. + for spec in args.messages: + module, path = spec.split(":", 1) + mod_dir = os.path.join(workdir, "src", module) + os.makedirs(mod_dir, exist_ok=True) + shutil.copy(path, os.path.join(mod_dir, "messages.txt")) + + # Track which .md files exist in man2 BEFORE the generator runs so + # we can skip them when running pandoc (they're module-level READMEs, + # not individual function man pages). + readme_stems = { + os.path.splitext(f)[0] for f in os.listdir(man2_md) if f.endswith(".md") + } + + # Run md_roff_compat.py from docs_dir so its relative paths + # (./md/man2, ../src) resolve correctly. + subprocess.run( + [sys.executable, os.path.join(scripts_dir, "md_roff_compat.py")], + cwd=docs_dir, + env={**os.environ, "PYTHONPATH": scripts_dir}, + check=True, + ) + + # Prepare output sub-directories inside the declared output trees. + cat_out = args.cat_out + html_out = args.html_out + roff_scratch = os.path.join(workdir, "roff") + for section in ["1", "2", "3"]: + os.makedirs(os.path.join(cat_out, f"cat{section}"), exist_ok=True) + os.makedirs(os.path.join(html_out, f"html{section}"), exist_ok=True) + os.makedirs(os.path.join(roff_scratch, f"man{section}")) + + # man1: straight from source .md files + _process_section( + pandoc, + man1_md, + os.path.join(roff_scratch, "man1"), + os.path.join(html_out, "html1"), + os.path.join(cat_out, "cat1"), + section_num=1, + ) + + # man2: only the per-function files generated by md_roff_compat.py + _process_section( + pandoc, + man2_md, + os.path.join(roff_scratch, "man2"), + os.path.join(html_out, "html2"), + os.path.join(cat_out, "cat2"), + section_num=2, + skip_stems=readme_stems, + ) + + # man3: all files written to man3 (messages-derived pages) + _process_section( + pandoc, + man3_md, + os.path.join(roff_scratch, "man3"), + os.path.join(html_out, "html3"), + os.path.join(cat_out, "cat3"), + section_num=3, + ) + + +if __name__ == "__main__": + main() diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index cc909308600..35322b157ac 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -1,13 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2026, The OpenROAD Authors +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//bazel:man_pages.bzl", "man_pages") man_pages( name = "man_pages", - docs_srcs = [ - "Makefile", - ] + glob(["md/**/*.md"]), + docs_srcs = glob(["md/**/*.md"]), messages = [ "//src/ant:messages_txt", "//src/cts:messages_txt", @@ -62,10 +61,14 @@ man_pages( "//src/utl:README.md", ], scripts = [ - "//docs/src/scripts:link_readmes.sh", "//docs/src/scripts:md_roff_compat.py", "//docs/src/scripts:manpage.py", "//docs/src/scripts:extract_utils.py", ], visibility = ["//visibility:public"], ) + +build_test( + name = "man_pages_build_test", + targets = [":man_pages"], +) diff --git a/docs/Makefile b/docs/Makefile index 9e5bb23e480..2cb466fbca9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -124,13 +124,8 @@ $(CAT2_DIR)/%.2: $(CAT2_DIR)/%.md $(CAT3_DIR)/%.3: $(CAT3_DIR)/%.md mv $< $@ -# Single-invocation target used by //docs:man_pages (Bazel). Chains the -# preprocessing step (README symlinks + roff-compat conversion) with the -# cat and web outputs that the Bazel rule copies into its declared outputs. -bazel-manpages: preprocess cat web - # Phony targets -.PHONY: all bazel-manpages preprocess +.PHONY: all preprocess # _____ _____ _ _ _____ _ ___ __ # / ____| __ \| | | |_ _| \ | \ \ / / diff --git a/pyproject.toml b/pyproject.toml index 8ccc33a2fb2..2fa1fb826f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ [tool.black] -extend-exclude = ''' -^/third-party/ -''' -force-exclude = ''' -^/third-party/ -''' +extend-exclude = '^/third-party/' +force-exclude = '^/third-party/'