diff --git a/.github/workflows/accelerate_doc.yml b/.github/workflows/accelerate_doc.yml index db1d1c64..62132787 100644 --- a/.github/workflows/accelerate_doc.yml +++ b/.github/workflows/accelerate_doc.yml @@ -38,7 +38,12 @@ jobs: run: uv venv --python 3.10 .venv - name: Install Python dependencies - run: uv pip install --python .venv/bin/python ./doc-builder './accelerate[dev]' + # light install: only accelerate's real (light) doc-build deps; torch & co are + # mocked at build time from the mock-deps registry + run: | + uv pip install --python .venv/bin/python ./doc-builder + uv pip install --python .venv/bin/python --no-deps ./accelerate + ./.venv/bin/doc-builder light-install accelerate - name: Make documentation run: | diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml index e698a8d5..abcc672e 100644 --- a/.github/workflows/build_main_documentation.yml +++ b/.github/workflows/build_main_documentation.yml @@ -46,12 +46,6 @@ on: convert_notebooks: type: boolean description: "Convert notebooks to markdown files before building docs." - # Docker image to use for the build. Set custom_container="huggingface/transformers-doc-builder" if you need - # a complete install. Default to `""` which means CI will run on the runner directly. - custom_container: - type: string - default: "" - description: "Docker image to use for the build." secrets: hf_token: required: true @@ -61,8 +55,6 @@ on: jobs: build_main_documentation: runs-on: ubuntu-22.04 - container: - ${{ inputs.custom_container }} defaults: run: shell: bash @@ -92,28 +84,7 @@ jobs: node-version: '20' cache-dependency-path: "kit/package-lock.json" - - name: Export ROOT_APT_GET ('apt-get' or 'sudo apt-get') - # Use `sudo` only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed) - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "ROOT_APT_GET=sudo apt-get" >> $GITHUB_ENV - else - echo "ROOT_APT_GET=apt-get" >> $GITHUB_ENV - fi - - - name: Export PIP_OR_UV ('pip' or 'uv pip') - # Use `uv` only if running on the base runner. - # When using a container, `pip` has already been used to installed existing deps - # and is therefore quicker to resolve (already have some cached stuff). - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "PIP_OR_UV=uv pip" >> $GITHUB_ENV - else - echo "PIP_OR_UV=pip" >> $GITHUB_ENV - fi + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Set env variables run: | @@ -134,13 +105,11 @@ jobs: # Needed to upload zips to hf.co - name: Install zip - # Use sudo only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed). - run: $ROOT_APT_GET install -y zip + run: sudo apt-get install -y zip - name: Install libgl1 if: inputs.install_libgl1 - run: $ROOT_APT_GET install -y libgl1 + run: sudo apt-get install -y libgl1 - name: Install Rust uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master 2026-03-27 @@ -148,32 +117,33 @@ jobs: with: toolchain: stable - # Use venv in both cases - - name: Install uv - run: | - pip install -U uv - uv venv - - name: Setup environment shell: bash run: | + uv venv source .venv/bin/activate - $PIP_OR_UV uninstall doc-builder cd doc-builder git pull origin main - $PIP_OR_UV install . + uv pip install . cd .. - if [[ -n "${{ inputs.package_path }}" ]] - then - cd ${{ inputs.package_path }} - $PIP_OR_UV install .[dev] - elif [[ "${{ inputs.additional_args }}" != *"--not_python_module"* ]]; + if [[ "${{ inputs.additional_args }}" == *"--not_python_module"* ]] then - cd ${{ inputs.package }} - $PIP_OR_UV install .[dev] + echo "Not a Python module; skipping package install." + else + package_dir="${{ inputs.package_path || inputs.package }}" + # Light install when the package has a mock-deps registry entry: only its + # real (light) doc-build deps are installed and the heavy ones are mocked + # at build time. Falls back to a full [dev] install otherwise. + if doc-builder light-install ${{ env.package_name }} --check + then + uv pip install "./$package_dir" --no-deps + doc-builder light-install ${{ env.package_name }} + else + echo "Falling back to a full install." + uv pip install "./$package_dir[dev]" + fi fi - cd .. - name: Setup git run: | @@ -244,9 +214,8 @@ jobs: HF_TOKEN: ${{ secrets.hf_token }} PACKAGE_NAME: ${{ env.package_name }} run: | - pip install -U huggingface_hub cd build_dir - hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc/$PACKAGE_NAME" + uvx --from huggingface_hub hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc/$PACKAGE_NAME" cd .. - name: Push to dataset (legacy, kept for rollback safety) diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 0b7f0c16..6cddcbb0 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -46,12 +46,6 @@ on: convert_notebooks: type: boolean description: "Convert notebooks to markdown files before building docs." - # Docker image to use for the build. Set custom_container="huggingface/transformers-doc-builder" if you need - # a complete install. Default to `""` which means CI will run on the runner directly. - custom_container: - type: string - default: "" - description: "Docker image to use for the build." # Debug purposes only! doc_builder_revision: type: string @@ -66,8 +60,6 @@ on: jobs: build_pr_documentation: runs-on: ubuntu-22.04 - container: - ${{ inputs.custom_container }} defaults: run: shell: bash @@ -102,28 +94,7 @@ jobs: node-version: '20' cache-dependency-path: "kit/package-lock.json" - - name: Export ROOT_APT_GET ('apt-get' or 'sudo apt-get') - # Use `sudo` only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed) - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "ROOT_APT_GET=sudo apt-get" >> $GITHUB_ENV - else - echo "ROOT_APT_GET=apt-get" >> $GITHUB_ENV - fi - - - name: Export PIP_OR_UV ('pip' or 'uv pip') - # Use `uv` only if running on the base runner. - # When using a container, `pip` has already been used to installed existing deps - # and is therefore quicker to resolve (already have some cached stuff). - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "PIP_OR_UV=uv pip" >> $GITHUB_ENV - else - echo "PIP_OR_UV=pip" >> $GITHUB_ENV - fi + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Set env variables run: | @@ -151,7 +122,7 @@ jobs: - name: Install libgl1 if: inputs.install_libgl1 - run: $ROOT_APT_GET install -y libgl1 + run: sudo apt-get install -y libgl1 - name: Install Rust uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master 2026-03-27 @@ -159,32 +130,33 @@ jobs: with: toolchain: stable - # Use venv in both cases - - name: Install uv - run: | - pip install -U uv - uv venv - - name: Setup environment shell: bash run: | + uv venv source .venv/bin/activate - $PIP_OR_UV uninstall doc-builder cd doc-builder git pull origin ${{ inputs.doc_builder_revision }} - $PIP_OR_UV install . + uv pip install . cd .. - if [[ -n "${{ inputs.package_path }}" ]] - then - cd ${{ inputs.package_path }} - $PIP_OR_UV install .[dev] - elif [[ "${{ inputs.additional_args }}" != *"--not_python_module"* ]]; + if [[ "${{ inputs.additional_args }}" == *"--not_python_module"* ]] then - cd ${{ inputs.package }} - $PIP_OR_UV install .[dev] + echo "Not a Python module; skipping package install." + else + package_dir="${{ inputs.package_path || inputs.package }}" + # Light install when the package has a mock-deps registry entry: only its + # real (light) doc-build deps are installed and the heavy ones are mocked + # at build time. Falls back to a full [dev] install otherwise. + if doc-builder light-install ${{ env.package_name }} --check + then + uv pip install "./$package_dir" --no-deps + doc-builder light-install ${{ env.package_name }} + else + echo "Falling back to a full install." + uv pip install "./$package_dir[dev]" + fi fi - cd .. - name: Run pre-command shell: bash diff --git a/.github/workflows/delete_old_pr_documentations.yml b/.github/workflows/delete_old_pr_documentations.yml index 35c5d494..b1e15605 100644 --- a/.github/workflows/delete_old_pr_documentations.yml +++ b/.github/workflows/delete_old_pr_documentations.yml @@ -16,8 +16,9 @@ jobs: - uses: denoland/setup-deno@11b63cf76cfcafb4e43f97b6cad24d8e8438f62d # v1.5.2 with: deno-version: v1.x + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Install hf CLI - run: pip install -U huggingface_hub + run: uv tool install huggingface_hub - run: deno run --allow-env --allow-net --allow-run --allow-read ./delete-old-prs.ts env: HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }} diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml index b93ebe67..5cb8073e 100644 --- a/.github/workflows/upload_pr_documentation.yml +++ b/.github/workflows/upload_pr_documentation.yml @@ -39,14 +39,13 @@ jobs: # Uncomment the following line to use a specific revision of doc-builder # ref: fix-corrupted-zip + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - name: Setup environment shell: bash id: setup-env run: | - pip install black - cd doc-builder - pip install . - cd .. + uv pip install --system ./doc-builder echo "current_work_dir=$(pwd)" >> $GITHUB_OUTPUT - name: 'Download artifact' @@ -137,8 +136,7 @@ jobs: HF_TOKEN: ${{ secrets.hf_token }} run: | cd build_dir - pip install -U huggingface_hub - hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc-dev/$PACKAGE_NAME" + uvx --from huggingface_hub hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc-dev/$PACKAGE_NAME" - name: Find doc comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 diff --git a/README.md b/README.md index b6ffc0ec..2a882bee 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,33 @@ A page is reused when its generated MDX (post autodoc and internal-link resoluti `--html_page_cache_write` also stores freshly built pages. Only enable it on trusted builds (the shared GitHub workflows enable it on main-branch builds and keep PR builds read-only), so that untrusted code cannot poison the cache. Cache failures are never fatal: any error degrades to building the affected pages. +### Building without installing heavy dependencies + +doc-builder imports the documented library to read docstrings with `inspect` — necessary because libraries like `transformers` construct docstrings dynamically. Historically that meant installing the library's full dependency tree (torch, GPU wheels, custom containers) just to build documentation. + +doc-builder mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed. Each library's registry file `src/doc_builder/mock_deps/.txt` records both sides of that split: + +- bare lines name the heavy dependencies to **mock** at build time (applied automatically by `doc-builder build`/`preview`); +- `real:` lines name the light dependencies a doc build must **install for real** (things the library imports unguarded, version-pinned requirements, packages used through `isinstance` checks, ...); +- `nodeps:` lines are also installed for real but with `--no-deps`, because their own dependency trees pull in heavy packages (e.g. `accelerate`, whose install requires torch). + +`doc-builder light-install ` installs the `real:`/`nodeps:` dependencies (via `uv`), so a full light setup is: + +```bash +uv pip install ./accelerate --no-deps +doc-builder light-install accelerate +doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test-build +# torch and deepspeed are mocked automatically (src/doc_builder/mock_deps/accelerate.txt) +``` + +Install the library before running `light-install`: bare registry names are pinned to the library's own declared version ranges (e.g. transformers' `tokenizers>=0.22,<=0.23`), so registry files never have to chase upstream pins. `light-install` exits with code 3 when the library has no registry entry — `light-install --check` tests this without installing anything, which is how the shared GitHub workflows decide between the light path and a full `[dev]` install. For a library without a registry entry (or to experiment), `--mock_deps torch,...` adds extra mocks on top of the registry. + +The registry key is the doc-builder *library name* as passed to `doc-builder build` — for namespace subpackages that is the dotted module name (e.g. `mock_deps/optimum.intel.txt`), and libraries that build under a shared name share a file (optimum-onnx builds as `optimum`). A registered name may itself be a dotted submodule (e.g. `optimum.onnxruntime`), which mocks a missing sibling inside an installed namespace package. + +The mocked packages are importable, pass the `importlib.util.find_spec` + `importlib.metadata` availability checks HF libraries use, are subclassable without affecting the real subclass's signature or docstring, behave as pass-through decorators, and render in signatures like the real objects (`repr(torch.float32) == "torch.float32"`). Packages that are actually installed are never mocked, and `doc-builder preview` supports the flag too. + +Fidelity, verified on `accelerate` built without torch installed: 46/48 pages byte-identical to a real-torch build; the two remaining pages differ only in typing paths rendered from the public import path instead of the internal one (e.g. `torch.nn.Module` instead of `torch.nn.modules.module.Module`). + ### Notebook conversion `doc-builder` can also automatically convert some of the documentation guides or tutorials into notebooks. This requires two steps: diff --git a/setup.py b/setup.py index e468a5a9..61a62151 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,8 @@ keywords="doc documentation doc-builder huggingface hugging face", url="https://github.com/huggingface/doc-builder", package_dir={"": "src"}, + package_data={"doc_builder": ["mock_deps/*.txt"]}, + include_package_data=True, packages=find_packages("src"), extras_require=extras, install_requires=install_requires, diff --git a/src/doc_builder/autodoc.py b/src/doc_builder/autodoc.py index 083cc85d..323b1b32 100644 --- a/src/doc_builder/autodoc.py +++ b/src/doc_builder/autodoc.py @@ -16,6 +16,7 @@ import inspect import json import re +import sys from functools import lru_cache from .convert_md_to_mdx import convert_md_docstring_to_mdx @@ -44,6 +45,16 @@ def find_object_in_package(object_name, package): full_module_path = f"{module.__name__}.{split}" importlib.import_module(full_module_path) submodule = getattr(module, split, None) + if submodule is None: + # a submodule can be importable without being bound as an + # attribute of its parent (e.g. lazy-module parents whose + # attribute binding was skipped because the submodule was + # already in sys.modules when first accessed) + submodule = sys.modules.get(full_module_path) + if submodule is not None: + # heal the missing binding so later attribute walks + # (e.g. get_shortest_path) resolve too + setattr(module, split, submodule) except ImportError: return None # Return None if the module cannot be found or imported module = submodule @@ -93,6 +104,12 @@ def get_type_name(typ): if isinstance(typ, type): # If it's a class, use its name. return getattr(typ, "__qualname__", None) or getattr(typ, "__name__", None) or str(typ) + # Non-classes carrying a name (e.g. mocked classes from `--mock_deps`) render like + # classes; typing constructs (whose reprs are the readable form) are excluded. + if getattr(typ, "__module__", "") != "typing": + qualname = getattr(typ, "__qualname__", None) + if isinstance(qualname, str): + return qualname return str(typ) # otherwise, trust its string representation diff --git a/src/doc_builder/commands/build.py b/src/doc_builder/commands/build.py index eb42581a..362961d3 100644 --- a/src/doc_builder/commands/build.py +++ b/src/doc_builder/commands/build.py @@ -23,6 +23,7 @@ from doc_builder import build_doc, update_versions_file from doc_builder.build_cache import PageCache, hash_kit_tree, page_cache_key +from doc_builder.mock_imports import get_registry_mock_deps, mock_deps from doc_builder.utils import ( get_default_branch_name, get_doc_config, @@ -148,6 +149,14 @@ def stage_kit_routes(kit_folder, tmp_dir, output_path): def build_command(args): + # heavy deps that can be mocked come from the per-library registry + # (doc_builder/mock_deps/.txt); --mock_deps adds extra ones. + # Must happen before the documented library is imported; packages that are + # actually installed are never mocked. + registry_deps = get_registry_mock_deps(args.library_name) + extra_deps = args.mock_deps.split(",") if args.mock_deps else [] + if registry_deps or extra_deps: + mock_deps(registry_deps + extra_deps) read_doc_config(args.path_to_docs) if args.html: # Error at the beginning if node is not properly installed. @@ -356,6 +365,14 @@ def build_command_parser(subparsers=None): "`hf://buckets/hf-doc-build/doc-build-cache`) or a local directory. Only used with --html. Pages whose " "generated mdx did not change are reused from the cache instead of being prerendered again.", ) + parser.add_argument( + "--mock_deps", + type=str, + default=None, + help="Comma-separated list of heavy dependencies (e.g. `torch,tensorflow`) to mock instead of importing, so " + "the documented library can be introspected without installing them. Packages that are actually installed " + "are never mocked.", + ) parser.add_argument( "--html_page_cache_write", action="store_true", diff --git a/src/doc_builder/commands/doc_builder_cli.py b/src/doc_builder/commands/doc_builder_cli.py index a8dea4a3..43a0959b 100644 --- a/src/doc_builder/commands/doc_builder_cli.py +++ b/src/doc_builder/commands/doc_builder_cli.py @@ -19,6 +19,7 @@ from doc_builder.commands.check_links import check_links_command_parser from doc_builder.commands.convert_doc_file import convert_command_parser from doc_builder.commands.embeddings import embeddings_command_parser +from doc_builder.commands.light_install import light_install_command_parser from doc_builder.commands.notebook_to_mdx import notebook_to_mdx_command_parser from doc_builder.commands.preview import preview_command_parser from doc_builder.commands.push import push_command_parser @@ -34,6 +35,7 @@ def main(): build_command_parser(subparsers=subparsers) check_links_command_parser(subparsers=subparsers) embeddings_command_parser(subparsers=subparsers) + light_install_command_parser(subparsers=subparsers) notebook_to_mdx_command_parser(subparsers=subparsers) style_command_parser(subparsers=subparsers) preview_command_parser(subparsers=subparsers) diff --git a/src/doc_builder/commands/light_install.py b/src/doc_builder/commands/light_install.py new file mode 100644 index 00000000..ac90daeb --- /dev/null +++ b/src/doc_builder/commands/light_install.py @@ -0,0 +1,136 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import re +import shlex +import subprocess +import sys + +from doc_builder.mock_imports import get_registry_real_deps, registry_file_path + + +def _library_requirements(library_name): + """ + The documented library's own declared requirements (marker-free ones), as a + mapping of normalized name -> version specifier string. Empty when the library + is not installed yet. + """ + from packaging.requirements import Requirement + + requirements = {} + for dist_name in (library_name, library_name.replace(".", "-")): + try: + requires = importlib.metadata.requires(dist_name) or [] + except importlib.metadata.PackageNotFoundError: + continue + for raw in requires: + try: + requirement = Requirement(raw) + except Exception: + continue + if requirement.marker is not None: + continue # extras & platform-conditional requirements + name = re.sub(r"[-_.]+", "-", requirement.name).lower() + if requirement.specifier: + requirements[name] = str(requirement.specifier) + break + return requirements + + +def _refine(specs, requirements): + """ + Pins bare registry names to the documented library's own declared version + range, so registry files don't have to chase pins like transformers' + `tokenizers>=0.22,<=0.23`. + """ + refined = [] + for spec in specs: + if re.fullmatch(r"[A-Za-z0-9._-]+", spec): + name = re.sub(r"[-_.]+", "-", spec).lower() + if name in requirements: + refined.append(spec + requirements[name]) + continue + refined.append(spec) + return refined + + +def light_install_command(args): + """ + Installs the real (non-mocked) dependencies a light docs build of `library_name` + needs, from the `real:`/`nodeps:` lines of its mock-deps registry file. The + documented library itself is not installed — install it first with `--no-deps` + (its heavy dependencies are mocked automatically at build time); bare registry + names are then pinned to the library's own declared version ranges. + + Exits with code 3 when the library has no registry entry, so callers can fall + back to a full install: + + if doc-builder light-install my_package --check; then + uv pip install ./my_package --no-deps + doc-builder light-install my_package + else + uv pip install "./my_package[dev]" + fi + """ + if not registry_file_path(args.library_name).is_file(): + print( + f"No mock-deps registry entry for `{args.library_name}` " + f"(expected {registry_file_path(args.library_name)}); do a full install instead.", + file=sys.stderr, + ) + sys.exit(3) + if args.check: + print(f"`{args.library_name}` has a mock-deps registry entry.") + return + resolved, nodeps = get_registry_real_deps(args.library_name) + requirements = _library_requirements(args.library_name) + # target the interpreter running doc-builder, whether or not a venv is "active" + uv_pip_install = ["uv", "pip", "install", "--python", sys.executable] + for specs, extra_flags in [(resolved, []), (nodeps, ["--no-deps"])]: + if not specs: + continue + specs = _refine(specs, requirements) + if args.dry_run: + print(" ".join(["uv", "pip", "install", *extra_flags, *(shlex.quote(spec) for spec in specs)])) + continue + subprocess.run([*uv_pip_install, *extra_flags, *specs], check=True) + if not (resolved or nodeps): + print(f"`{args.library_name}` needs no real dependencies beyond the library itself.") + + +def light_install_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("light-install") + else: + from argparse import ArgumentParser + + parser = ArgumentParser("Doc Builder light-install command") + parser.add_argument( + "library_name", type=str, help="Library whose registered real doc-build dependencies to install." + ) + parser.add_argument( + "--check", + action="store_true", + help="Only check whether the library has a registry entry (exit code 3 when it does not).", + ) + parser.add_argument( + "--dry_run", + dest="dry_run", + action="store_true", + help="Print the install commands instead of running them.", + ) + if subparsers is not None: + parser.set_defaults(func=light_install_command) + return parser diff --git a/src/doc_builder/commands/preview.py b/src/doc_builder/commands/preview.py index 8c26c922..0c5c66e2 100644 --- a/src/doc_builder/commands/preview.py +++ b/src/doc_builder/commands/preview.py @@ -23,6 +23,7 @@ from doc_builder import build_doc from doc_builder.commands.build import check_node_is_available, docs_node_env, run_npm, stage_kit_routes from doc_builder.commands.convert_doc_file import find_root_git +from doc_builder.mock_imports import get_registry_mock_deps, mock_deps from doc_builder.utils import ( is_watchdog_available, locate_kit_folder, @@ -137,6 +138,14 @@ def start_sveltekit_dev(kit_dir, env): def preview_command(args): + # heavy deps that can be mocked come from the per-library registry + # (doc_builder/mock_deps/.txt); --mock_deps adds extra ones. + # Must happen before the documented library is imported; packages that are + # actually installed are never mocked. + registry_deps = get_registry_mock_deps(args.library_name) + extra_deps = args.mock_deps.split(",") if args.mock_deps else [] + if registry_deps or extra_deps: + mock_deps(registry_deps + extra_deps) if not is_watchdog_available(): raise ImportError( "Please install `watchdog` to run `doc-builder preview` command.\nYou can do so through pip: `pip install watchdog`" @@ -216,6 +225,14 @@ def preview_command_parser(subparsers=None): action="store_true", help="Whether docs files do NOT have corresponding python module (like HF course & hub docs).", ) + parser.add_argument( + "--mock_deps", + type=str, + default=None, + help="Comma-separated list of heavy dependencies (e.g. `torch,tensorflow`) to mock instead of importing, so " + "the documented library can be introspected without installing them. Packages that are actually installed " + "are never mocked.", + ) if subparsers is not None: parser.set_defaults(func=preview_command) diff --git a/src/doc_builder/mock_deps/accelerate.txt b/src/doc_builder/mock_deps/accelerate.txt new file mode 100644 index 00000000..6b30aff4 --- /dev/null +++ b/src/doc_builder/mock_deps/accelerate.txt @@ -0,0 +1,11 @@ +# Heavy dependencies mocked when building accelerate docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +deepspeed +torchvision +real:numpy +real:packaging +real:psutil +real:pyyaml +real:safetensors +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/autotrain.txt b/src/doc_builder/mock_deps/autotrain.txt new file mode 100644 index 00000000..25f22d7a --- /dev/null +++ b/src/doc_builder/mock_deps/autotrain.txt @@ -0,0 +1,19 @@ +# Heavy dependencies mocked when building autotrain docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +loguru +real:pandas +real:pyyaml +real:requests +real:huggingface_hub +nodeps:accelerate +real:safetensors +matplotlib +tensorflow +flax +jax +torchvision +torchaudio +real:transformers +real:datasets +real:scikit-learn diff --git a/src/doc_builder/mock_deps/bitsandbytes.txt b/src/doc_builder/mock_deps/bitsandbytes.txt new file mode 100644 index 00000000..6a2b25b3 --- /dev/null +++ b/src/doc_builder/mock_deps/bitsandbytes.txt @@ -0,0 +1,5 @@ +# Heavy dependencies mocked when building bitsandbytes docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +real:numpy +real:packaging diff --git a/src/doc_builder/mock_deps/datasets.txt b/src/doc_builder/mock_deps/datasets.txt new file mode 100644 index 00000000..8776ec79 --- /dev/null +++ b/src/doc_builder/mock_deps/datasets.txt @@ -0,0 +1,15 @@ +# datasets docs build without mocking: its framework integrations are optional +# and its required deps (pyarrow, pandas, ...) must be really installed. +real:numpy +real:pyarrow +real:dill +real:pandas +real:requests +real:tqdm +real:xxhash +real:multiprocess +real:fsspec +real:aiohttp +real:huggingface_hub +real:packaging +real:pyyaml diff --git a/src/doc_builder/mock_deps/diffusers.txt b/src/doc_builder/mock_deps/diffusers.txt new file mode 100644 index 00000000..24e9e678 --- /dev/null +++ b/src/doc_builder/mock_deps/diffusers.txt @@ -0,0 +1,12 @@ +# Heavy dependencies mocked when building diffusers docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +real:importlib_metadata +real:filelock +real:huggingface_hub +real:numpy +real:regex +real:requests +real:safetensors +real:pillow +real:transformers diff --git a/src/doc_builder/mock_deps/evaluate.txt b/src/doc_builder/mock_deps/evaluate.txt new file mode 100644 index 00000000..8a57dc65 --- /dev/null +++ b/src/doc_builder/mock_deps/evaluate.txt @@ -0,0 +1,15 @@ +# Heavy dependencies mocked when building evaluate docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +matplotlib +real:datasets +real:numpy +real:dill +real:pandas +real:requests +real:tqdm +real:xxhash +real:multiprocess +real:fsspec +real:huggingface_hub +real:packaging diff --git a/src/doc_builder/mock_deps/huggingface_hub.txt b/src/doc_builder/mock_deps/huggingface_hub.txt new file mode 100644 index 00000000..ff513a4c --- /dev/null +++ b/src/doc_builder/mock_deps/huggingface_hub.txt @@ -0,0 +1,10 @@ +# Heavy dependencies mocked when building huggingface_hub docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +real:filelock +real:fsspec +real:packaging +real:pyyaml +real:requests +real:tqdm +real:typing_extensions diff --git a/src/doc_builder/mock_deps/kernels.txt b/src/doc_builder/mock_deps/kernels.txt new file mode 100644 index 00000000..771d86cb --- /dev/null +++ b/src/doc_builder/mock_deps/kernels.txt @@ -0,0 +1,8 @@ +# Dependencies for building kernels docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:pyyaml +real:huggingface_hub +real:packaging +real:tomli +kernels_data diff --git a/src/doc_builder/mock_deps/lerobot.txt b/src/doc_builder/mock_deps/lerobot.txt new file mode 100644 index 00000000..eb71c5d7 --- /dev/null +++ b/src/doc_builder/mock_deps/lerobot.txt @@ -0,0 +1,7 @@ +# Dependencies for building lerobot docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:numpy +real:pyyaml +real:huggingface_hub +real:packaging diff --git a/src/doc_builder/mock_deps/lighteval.txt b/src/doc_builder/mock_deps/lighteval.txt new file mode 100644 index 00000000..eddd1e21 --- /dev/null +++ b/src/doc_builder/mock_deps/lighteval.txt @@ -0,0 +1,27 @@ +# Heavy dependencies mocked when building lighteval docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +vllm +matplotlib +ray +more_itertools +real:transformers +real:datasets +real:huggingface_hub +real:pydantic +real:typer +real:rich +real:colorama +real:termcolor +real:tabulate +real:aenum +real:nltk +real:pytablewriter +real:sacrebleu +real:inspect_ai +real:latex2sympy2_extended==1.0.6 +real:sympy +real:scikit-learn +real:pandas +nodeps:accelerate +real:scipy diff --git a/src/doc_builder/mock_deps/openenv.txt b/src/doc_builder/mock_deps/openenv.txt new file mode 100644 index 00000000..fe2422f4 --- /dev/null +++ b/src/doc_builder/mock_deps/openenv.txt @@ -0,0 +1,13 @@ +# Heavy dependencies mocked when building openenv docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +tomli_w +fastapi +fastmcp +mcp +openai +gradio +websockets +real:requests +real:pydantic +real:aiohttp +real:typer diff --git a/src/doc_builder/mock_deps/optimum.executorch.txt b/src/doc_builder/mock_deps/optimum.executorch.txt new file mode 100644 index 00000000..dd81e75f --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.executorch.txt @@ -0,0 +1,6 @@ +# optimum-executorch docs build without mocks (validated with only the library installed). +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/optimum.habana.txt b/src/doc_builder/mock_deps/optimum.habana.txt new file mode 100644 index 00000000..a279fbda --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.habana.txt @@ -0,0 +1,14 @@ +# Heavy dependencies mocked when building optimum.habana docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +# NOTE: also requires version-pinned real deps (transformers, accelerate, diffusers, +# sentence-transformers) and a `habana-torch-plugin` distribution visible to `pip list`. +torch +sklearn +scipy +nodeps:optimum +real:transformers==4.55.* +nodeps:accelerate==1.10.1 +real:diffusers==0.35.1 +nodeps:sentence-transformers==3.3.1 +real:sentencepiece +real:protobuf diff --git a/src/doc_builder/mock_deps/optimum.intel.txt b/src/doc_builder/mock_deps/optimum.intel.txt new file mode 100644 index 00000000..c77174c2 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.intel.txt @@ -0,0 +1,10 @@ +# Heavy dependencies mocked when building optimum.intel docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +openvino +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +real:diffusers diff --git a/src/doc_builder/mock_deps/optimum.neuron.txt b/src/doc_builder/mock_deps/optimum.neuron.txt new file mode 100644 index 00000000..6aac8e08 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.neuron.txt @@ -0,0 +1,18 @@ +# Heavy dependencies mocked when building optimum.neuron docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +neuronx_distributed +torch_xla +torch_neuronx +libneuronxla +neuronxcc +peft +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +real:diffusers==0.35.* +real:sentencepiece +nodeps:accelerate +real:datasets diff --git a/src/doc_builder/mock_deps/optimum.tpu.txt b/src/doc_builder/mock_deps/optimum.tpu.txt new file mode 100644 index 00000000..d6c869f0 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.tpu.txt @@ -0,0 +1,6 @@ +# optimum-tpu docs build without mocks (validated with only the library installed). +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/optimum.txt b/src/doc_builder/mock_deps/optimum.txt new file mode 100644 index 00000000..eba30697 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.txt @@ -0,0 +1,13 @@ +# Heavy dependencies mocked when building optimum docs (see mock_imports.py). +# Shared by optimum-onnx, whose docs also build under the library name `optimum` +# (only --repo_name differs): onnxruntime/onnx belong to that build. +# Packages that are actually installed are never mocked. +torch +onnxruntime +onnx +nodeps:optimum-onnx +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +nodeps:optimum diff --git a/src/doc_builder/mock_deps/peft.txt b/src/doc_builder/mock_deps/peft.txt new file mode 100644 index 00000000..bacebf61 --- /dev/null +++ b/src/doc_builder/mock_deps/peft.txt @@ -0,0 +1,12 @@ +# Heavy dependencies mocked when building peft docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +real:numpy +real:packaging +real:psutil +real:pyyaml +real:transformers +real:tqdm +real:safetensors +real:huggingface_hub +nodeps:accelerate diff --git a/src/doc_builder/mock_deps/safetensors.txt b/src/doc_builder/mock_deps/safetensors.txt new file mode 100644 index 00000000..3b04f8ca --- /dev/null +++ b/src/doc_builder/mock_deps/safetensors.txt @@ -0,0 +1,7 @@ +# Heavy dependencies mocked when building safetensors docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +paddle +torch +tensorflow +jax +real:numpy diff --git a/src/doc_builder/mock_deps/setfit.txt b/src/doc_builder/mock_deps/setfit.txt new file mode 100644 index 00000000..5f41ea21 --- /dev/null +++ b/src/doc_builder/mock_deps/setfit.txt @@ -0,0 +1,13 @@ +# Heavy dependencies mocked when building setfit docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +sentence_transformers +joblib +sklearn +matplotlib +tensorflow +flax +jax +torchvision +torchaudio +scipy diff --git a/src/doc_builder/mock_deps/timm.txt b/src/doc_builder/mock_deps/timm.txt new file mode 100644 index 00000000..074d2ae3 --- /dev/null +++ b/src/doc_builder/mock_deps/timm.txt @@ -0,0 +1,8 @@ +# Heavy dependencies mocked when building timm docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +torchvision +real:pyyaml +real:huggingface_hub +real:safetensors +real:numpy diff --git a/src/doc_builder/mock_deps/tokenizers.txt b/src/doc_builder/mock_deps/tokenizers.txt new file mode 100644 index 00000000..c15de9a0 --- /dev/null +++ b/src/doc_builder/mock_deps/tokenizers.txt @@ -0,0 +1,4 @@ +# Dependencies for building tokenizers docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/trackio.txt b/src/doc_builder/mock_deps/trackio.txt new file mode 100644 index 00000000..2d5da55a --- /dev/null +++ b/src/doc_builder/mock_deps/trackio.txt @@ -0,0 +1,11 @@ +# Heavy dependencies mocked when building trackio docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +matplotlib +real:pandas +real:pydantic +real:huggingface_hub +gradio_client +orjson +uvicorn +starlette diff --git a/src/doc_builder/mock_deps/transformers.txt b/src/doc_builder/mock_deps/transformers.txt new file mode 100644 index 00000000..cc918d25 --- /dev/null +++ b/src/doc_builder/mock_deps/transformers.txt @@ -0,0 +1,24 @@ +# Heavy dependencies mocked when building transformers docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +tensorflow +flax +jax +torchvision +torchaudio +timm +deepspeed +real:filelock +real:huggingface_hub +real:numpy +real:packaging +real:pyyaml +real:regex +real:requests +real:tokenizers +real:safetensors +real:tqdm +real:sentencepiece +real:protobuf +real:pillow +real:rich diff --git a/src/doc_builder/mock_deps/trl.txt b/src/doc_builder/mock_deps/trl.txt new file mode 100644 index 00000000..c6925132 --- /dev/null +++ b/src/doc_builder/mock_deps/trl.txt @@ -0,0 +1,10 @@ +# Heavy dependencies mocked when building trl docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +real:datasets +real:transformers +real:rich +nodeps:accelerate +real:jinja2 +real:psutil +nodeps:peft diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py new file mode 100644 index 00000000..1e6baee6 --- /dev/null +++ b/src/doc_builder/mock_imports.py @@ -0,0 +1,409 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Mock heavy dependencies (torch, tensorflow, ...) so that a documented library can be +imported — and introspected with `inspect` — without installing them. + +doc-builder needs to import the documented library because docstrings are often built +dynamically (e.g. transformers' `add_start_docstrings` decorators), which rules out +purely static analysis. But importing e.g. `transformers` or `accelerate` requires +torch and friends, which are slow to install and may need custom containers (GPU +wheels, compiled extensions) just to build documentation. + +`mock_deps(["torch", ...])` installs a `sys.meta_path` finder that provides, for each +mocked package and any of its submodules: + +- importable module objects whose attributes are auto-created **mock classes**, so + `from torch import nn` and `class Model(nn.Module)` work; +- pass-through decorator behavior: calling a mock with a single function/class returns + it unchanged, so `@torch.no_grad()`-style decorators don't swallow the docstrings + that doc-builder is trying to read (same heuristic as Sphinx's `autodoc_mock_imports`); +- a `repr` equal to the dotted path (`repr(torch.float32) == "torch.float32"`), so + mocked default values render in signatures exactly like the real ones; +- fake distribution metadata, so availability checks in HF libraries — typically + `importlib.util.find_spec(pkg)` plus `importlib.metadata.metadata(pkg)` — treat the + mocked package as installed instead of routing to dummy objects. + +The documented library itself (and its light dependencies) must still be really +installed: `inspect` reads the real source, docstrings and signatures from it. +""" + +import abc +import importlib.abc +import importlib.machinery +import importlib.metadata +import sys +import types +import typing +from pathlib import Path + +MOCK_VERSION = "9999.0.0" + + +class _MockVersionString(str): + """ + Version string comparable against tuples, like torch's `TorchVersion` + (libraries write e.g. `torch.__version__ >= (2, 6)`). + """ + + def _as_tuple(self): + return tuple(int(p) for p in self.split(".") if p.isdigit()) + + def _coerce(self, other): + if isinstance(other, tuple): + return tuple(int(p) if not isinstance(p, int) else p for p in other) + return None + + def __ge__(self, other): + coerced = self._coerce(other) + return self._as_tuple() >= coerced if coerced is not None else str.__ge__(self, other) + + def __gt__(self, other): + coerced = self._coerce(other) + return self._as_tuple() > coerced if coerced is not None else str.__gt__(self, other) + + def __le__(self, other): + coerced = self._coerce(other) + return self._as_tuple() <= coerced if coerced is not None else str.__le__(self, other) + + def __lt__(self, other): + coerced = self._coerce(other) + return self._as_tuple() < coerced if coerced is not None else str.__lt__(self, other) + + +class _MockObject: + """ + A mock value for a dotted path (e.g. `torch.float32`): attribute access chains, + calls behave as pass-through decorators, `repr` is the dotted path, and using it + as a base class substitutes a plain-`type` base (PEP 560 `__mro_entries__`), so + real subclasses keep a normal metaclass and `inspect.signature` reads their real + `__init__` instead of a mock's. + """ + + def __init__(self, display_name): + object.__setattr__(self, "__display_name__", display_name) + # annotations render via `__qualname__`/`__name__` (e.g. `torch.nn.Module` + # in a signature shows as `Module`, like the real class). Instance attributes + # shadow the class's own `__qualname__` string. + short_name = display_name.rsplit(".", 1)[-1] + object.__setattr__(self, "__name__", short_name) + object.__setattr__(self, "__qualname__", short_name) + # tools rendering type names as `__module__.__qualname__` (e.g. transformers' + # auto_docstring) must see the mocked path, not this module's + parent = display_name.rsplit(".", 1)[0] if "." in display_name else display_name + object.__setattr__(self, "__module__", parent) + + def __repr__(self): + return self.__display_name__ + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _make_mock(f"{self.__display_name__}.{name}") + + def __call__(self, *args, **kwargs): + # pass decorated functions/classes through unchanged so their docstrings and + # signatures stay intact (a mock used as a decorator, e.g. `@torch.no_grad()`) + if len(args) == 1 and not kwargs and (callable(args[0]) or isinstance(args[0], type)): + return args[0] + return _make_mock(f"{self.__display_name__}()") + + def __mro_entries__(self, bases): + return (_make_base_class(self.__display_name__),) + + # PEP 604 unions in runtime-evaluated annotations (`torch.Tensor | None`): build a + # *real* typing.Union (mocks pass `typing._type_check` because they are callable), + # so tools that unwrap Optional/Union via typing.get_origin/get_args — like + # transformers' auto_docstring — treat it exactly like the real annotation + def __or__(self, other): + try: + return typing.Union[self, other] # noqa: UP007 + except TypeError: + return _make_mock(f"typing.Union[{self.__display_name__}, {_annotation_name(other)}]") + + def __ror__(self, other): + try: + return typing.Union[other, self] # noqa: UP007 + except TypeError: + return _make_mock(f"typing.Union[{_annotation_name(other)}, {self.__display_name__}]") + + def __getitem__(self, item): + # subscripted generics on mocks (`torch.Tensor[int]`-style, rare) + return self + + # keep common import-time expressions working + def __len__(self): + return 0 + + def __iter__(self): + return iter(()) + + def __contains__(self, item): + return False + + def __index__(self): + return 0 + + def __bool__(self): + return True + + def __eq__(self, other): + return self is other + + def __hash__(self): + return hash(self.__display_name__) + + # `issubclass(cls, torch.Tensor)` / `isinstance(x, torch.Tensor)` against a mock + # would be a TypeError (mocks are instances, not classes); scipy>=1.17 probes + # exactly this way when it sees `torch` in sys.modules. Nothing real is a + # subclass/instance of a mocked type, so answer False. + def __subclasscheck__(self, subclass): + return False + + def __instancecheck__(self, instance): + return False + + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return True + + def __ge__(self, other): + return True + + +def _annotation_name(obj): + """Renders `obj` the way `typing` renders it inside a union.""" + display_name = getattr(obj, "__display_name__", None) + if display_name is not None: + return display_name + if isinstance(obj, type): + module = getattr(obj, "__module__", "") + qualname = getattr(obj, "__qualname__", str(obj)) + return qualname if module in ("builtins", "") else f"{module}.{qualname}" + return str(obj) + + +_mock_cache = {} +_base_class_cache = {} + + +def _make_mock(display_name): + """Creates (and caches) a mock object for a dotted path.""" + if display_name not in _mock_cache: + _mock_cache[display_name] = _MockObject(display_name) + return _mock_cache[display_name] + + +class _MockBaseMeta(abc.ABCMeta): + """ + Metaclass for mock base classes providing *class-level* attribute fallback: real + subclasses may rely on attributes the real base would provide (e.g. a model class + without its own `forward` inherits `torch.nn.Module.forward`). It intentionally + does NOT define `__call__`, so `inspect.signature` on subclasses still reads their + real `__init__` (a metaclass `__call__` would shadow it). Deriving from `ABCMeta` + keeps mock bases combinable with real ABC bases without metaclass conflicts. + """ + + # only fall back for methods commonly inherited from framework bases and + # documented as such (e.g. `[[autodoc]] SomeModel - forward` where the model + # doesn't override nn.Module.forward). A blanket fallback would make + # `hasattr(cls, anything)` true and break libraries' own attribute checks + # (e.g. peft's prefix-registration validation). + _FALLBACK_ATTRIBUTES = {"forward", "call"} + + def __getattr__(cls, name): + if name not in _MockBaseMeta._FALLBACK_ATTRIBUTES: + raise AttributeError(name) + base_display_name = getattr(cls, "__display_name__", cls.__name__) + return _make_mock(f"{base_display_name}.{name}") + + +def _make_base_class(display_name): + """ + Creates (and caches) the class substituted when a mock is used as a base class + (`class Model(nn.Module)`). It provides permissive `__init__` and instance/class + attribute fallbacks while keeping subclass introspection intact. + """ + if display_name not in _base_class_cache: + _base_class_cache[display_name] = _MockBaseMeta( + display_name.rsplit(".", 1)[-1], + (object,), + { + "__display_name__": display_name, + "__module__": display_name.rsplit(".", 1)[0] if "." in display_name else display_name, + "__init__": lambda self, *args, **kwargs: None, + "__getattr__": lambda self, name: _make_mock(f"{display_name}.{name}"), + "__init_subclass__": classmethod(lambda cls, **kwargs: None), + "__class_getitem__": classmethod(lambda cls, item: cls), + }, + ) + return _base_class_cache[display_name] + + +class _MockModule(types.ModuleType): + """ + Mock module supporting the same expression surface as `_MockObject`: once code runs + `import torch.X`, the import system binds a real module object as the `X` attribute + of `torch` (shadowing the mock), and that module may then appear in runtime + expressions like PEP 604 annotations (`torch.Tensor | None` when `torch.Tensor` was + imported as a module path) or be called/subclassed. + """ + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _make_mock(f"{self.__name__}.{name}") + + def __call__(self, *args, **kwargs): + return _make_mock(f"{self.__name__}")(*args, **kwargs) + + def __or__(self, other): + return _make_mock(f"{self.__name__}").__or__(other) + + def __ror__(self, other): + return _make_mock(f"{self.__name__}").__ror__(other) + + def __mro_entries__(self, bases): + return (_make_base_class(self.__name__),) + + def __getitem__(self, item): + return self + + +class _MockDistribution(importlib.metadata.Distribution): + """Just enough metadata for `importlib.metadata.metadata/version` to succeed.""" + + def __init__(self, name): + self._name = name + + def read_text(self, filename): + if filename == "METADATA": + return f"Metadata-Version: 2.1\nName: {self._name}\nVersion: {MOCK_VERSION}\n" + return None + + def locate_file(self, path): + return None + + +class MockFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """Meta path finder+loader serving mock modules and metadata for mocked packages.""" + + def __init__(self, packages): + self.packages = set(packages) + + def _is_mocked(self, fullname): + # prefix matching so a registered name can be a submodule of a namespace + # package (e.g. mock `optimum.onnxruntime` while the real `optimum` and + # `optimum.intel` stay importable) + return any(fullname == package or fullname.startswith(package + ".") for package in self.packages) + + def find_spec(self, fullname, path=None, target=None): + if not self._is_mocked(fullname): + return None + spec = importlib.machinery.ModuleSpec(fullname, self, is_package=True) + return spec + + def create_module(self, spec): + module = _MockModule(spec.name) + module.__version__ = _MockVersionString(MOCK_VERSION) + module.__path__ = [] + return module + + def exec_module(self, module): + pass + + def find_distributions(self, context=importlib.metadata.DistributionFinder.Context()): + name = context.name + if name is None: + return [] + # normalize the distribution name the way importlib.metadata does + if name.replace("-", "_").lower() in {p.replace("-", "_").lower() for p in self.packages}: + return [_MockDistribution(name)] + return [] + + +def registry_file_path(library_name): + """Path of a documented library's registry file (which may not exist).""" + return Path(__file__).parent / "mock_deps" / f"{library_name}.txt" + + +def _read_registry(library_name): + registry_file = registry_file_path(library_name) + if not registry_file.is_file(): + return [] + lines = [] + for line in registry_file.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line: + lines.append(line) + return lines + + +def get_registry_mock_deps(library_name): + """ + Returns the registered list of mockable heavy dependencies for a documented + library, from `doc_builder/mock_deps/.txt` (one import name per + line, `#` comments). Lines prefixed with `real:` or `nodeps:` list packages to + install for real instead (see `get_registry_real_deps`) and are skipped here. + Empty when the library has no registry entry. + """ + return [line for line in _read_registry(library_name) if not line.startswith(("real:", "nodeps:"))] + + +def get_registry_real_deps(library_name): + """ + Returns the registered dependencies a light docs build must install for real + (not mock), as a tuple `(resolved, nodeps)` of pip requirement lists: + `real:` lines resolve normally, `nodeps:` lines must be installed + with `--no-deps` (their own dependency trees pull in heavy packages). + """ + resolved, nodeps = [], [] + for line in _read_registry(library_name): + if line.startswith("real:"): + resolved.append(line[len("real:") :].strip()) + elif line.startswith("nodeps:"): + nodeps.append(line[len("nodeps:") :].strip()) + return resolved, nodeps + + +def mock_deps(packages): + """ + Makes `packages` (e.g. `["torch"]`) importable as mocks. Must be called before the + documented library is imported. No-op for packages that are actually installed + (the real package takes precedence since the finder is appended last... but + availability must be consistent, so mocked packages that are really installed are + skipped explicitly). + """ + packages = [p.strip() for p in packages if p.strip()] + to_mock = [] + for package in packages: + try: + real = importlib.util.find_spec(package) + except (ImportError, ValueError): + real = None + if real is not None: + print(f"[mock-deps] `{package}` is actually installed, not mocking it") + continue + to_mock.append(package) + if to_mock: + finder = MockFinder(to_mock) + sys.meta_path.insert(0, finder) + print(f"[mock-deps] mocking imports for: {', '.join(sorted(to_mock))}") + return to_mock diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py new file mode 100644 index 00000000..6ceb9d04 --- /dev/null +++ b/tests/test_mock_imports.py @@ -0,0 +1,164 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import importlib.util +import inspect +import sys +import unittest + +from doc_builder.mock_imports import MOCK_VERSION, MockFinder, mock_deps + +PKG = "doc_builder_test_mocked_pkg" + + +class MockImportsTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.finder = MockFinder([PKG]) + sys.meta_path.insert(0, cls.finder) + + @classmethod + def tearDownClass(cls): + sys.meta_path.remove(cls.finder) + for name in list(sys.modules): + if name.startswith(PKG): + del sys.modules[name] + + def test_mocked_module_imports(self): + module = importlib.import_module(PKG) + self.assertEqual(module.__version__, MOCK_VERSION) + submodule = importlib.import_module(f"{PKG}.sub.module") + self.assertIsNotNone(submodule) + + def test_availability_checks_pass(self): + # the two checks HF libraries typically combine + self.assertIsNotNone(importlib.util.find_spec(PKG)) + self.assertIsNotNone(importlib.metadata.metadata(PKG)) + self.assertEqual(importlib.metadata.version(PKG), MOCK_VERSION) + + def test_mock_repr_and_name(self): + module = importlib.import_module(PKG) + mock = module.nn.Module + self.assertEqual(repr(mock), f"{PKG}.nn.Module") + self.assertEqual(mock.__name__, "Module") + self.assertEqual(mock.__qualname__, "Module") + + def test_subclass_keeps_real_signature(self): + module = importlib.import_module(PKG) + + class Model(module.nn.Module): + """A real class inheriting from a mocked base.""" + + def __init__(self, hidden_size: int = 8, dropout: float = 0.1): + pass + + self.assertEqual(list(inspect.signature(Model).parameters), ["hidden_size", "dropout"]) + self.assertEqual(Model.__doc__, "A real class inheriting from a mocked base.") + # the metaclass must not define `__call__`: it would shadow the subclass's + # real `__init__` under `inspect.signature` + self.assertNotIn("__call__", type(Model).__dict__) + + def test_mock_decorator_passes_through(self): + module = importlib.import_module(PKG) + + @module.no_grad() + def documented(a: int, b: str = "x"): + """Docstring survives mocked decorators.""" + + self.assertEqual(documented.__doc__, "Docstring survives mocked decorators.") + self.assertEqual(list(inspect.signature(documented).parameters), ["a", "b"]) + + def test_pep604_unions_are_real_typing_unions(self): + import typing + + module = importlib.import_module(PKG) + union = module.Tensor | None + # a real typing.Union, so tools unwrapping Optional via get_origin/get_args + # (e.g. transformers' auto_docstring) treat it like the real annotation + self.assertIs(typing.get_origin(union), typing.Union) + args = typing.get_args(union) + self.assertIs(args[1], type(None)) + self.assertEqual(getattr(args[0], "__module__", None), PKG) + self.assertEqual(getattr(args[0], "__qualname__", None), "Tensor") + # reflected form with a real type on the left + union = int | module.Tensor + self.assertIs(typing.get_origin(union), typing.Union) + + def test_subclass_inherits_mock_class_attributes(self): + module = importlib.import_module(PKG) + + class Model(module.nn.Module): + pass + + # class-level fallback: the real base (e.g. nn.Module) would provide `forward` + self.assertIsNotNone(getattr(Model, "forward", None)) + + def test_isinstance_issubclass_against_mock(self): + module = importlib.import_module(PKG) + # scipy>=1.17 does `issubclass(cls, torch.Tensor)` when torch is in + # sys.modules; a mock must answer False, not raise TypeError + self.assertFalse(issubclass(int, module.Tensor)) + self.assertFalse(isinstance(3, module.Tensor)) + + def test_dotted_submodule_mock(self): + # a registered dotted name mocks a missing submodule of a real (namespace) + # package, like `optimum.onnxruntime` when only `optimum` is installed + finder = MockFinder(["unittest.doc_builder_missing_sibling"]) + sys.meta_path.insert(0, finder) + try: + module = importlib.import_module("unittest.doc_builder_missing_sibling.sub") + self.assertEqual(module.__name__, "unittest.doc_builder_missing_sibling.sub") + import unittest as real_unittest + + # the real parent package is untouched + self.assertIs(real_unittest.TestCase, unittest.TestCase) + finally: + sys.meta_path.remove(finder) + for name in list(sys.modules): + if name.startswith("unittest.doc_builder_missing_sibling"): + del sys.modules[name] + + def test_registry_mock_deps(self): + from doc_builder.mock_imports import get_registry_mock_deps + + deps = get_registry_mock_deps("transformers") + self.assertIn("torch", deps) + self.assertIn("tensorflow", deps) + # comments are stripped + self.assertTrue(all(not d.startswith("#") for d in deps)) + # unknown library -> empty + self.assertEqual(get_registry_mock_deps("not-a-real-library"), []) + # validated as needing no mocks -> empty but present + self.assertEqual(get_registry_mock_deps("datasets"), []) + + def test_registry_real_deps(self): + from doc_builder.mock_imports import get_registry_mock_deps, get_registry_real_deps + + resolved, nodeps = get_registry_real_deps("peft") + self.assertIn("transformers", resolved) + self.assertIn("accelerate", nodeps) # accelerate's own deps include torch + # real:/nodeps: lines are install directives, not mocks + mocks = get_registry_mock_deps("peft") + self.assertIn("torch", mocks) + self.assertFalse(any(m.startswith(("real:", "nodeps:")) for m in mocks)) + self.assertEqual(get_registry_real_deps("not-a-real-library"), ([], [])) + + def test_installed_packages_are_not_mocked(self): + # `json` is really installed, so it must be skipped + mocked = mock_deps(["json"]) + self.assertEqual(mocked, []) + import json + + self.assertTrue(json.dumps({"a": 1}))