From 98905fe061e706912a44fc5b537df4961bcef66a Mon Sep 17 00:00:00 2001 From: Mishig Date: Sun, 5 Jul 2026 21:24:14 +0200 Subject: [PATCH 1/5] feat: --mock_deps to build docs without installing heavy dependencies doc-builder must import the documented library (docstrings are built dynamically, e.g. transformers' add_start_docstrings decorators), which historically forced installing the full dependency tree - torch, GPU wheels, custom containers - just to build documentation. --mock_deps torch,... installs a sys.meta_path finder providing, for each mocked package and its submodules: - importable module trees that pass the find_spec + importlib.metadata availability checks HF libraries gate exports on (naive sys.modules mocks fail these and route to dummy objects) - mocks usable as base classes via PEP 560 __mro_entries__, which substitutes a plain-`type` base so real subclasses keep their real __init__ signature and docstring under inspect - pass-through decorator behavior (calling a mock with a single function/class returns it unchanged), so decorated docstrings survive - dotted-path repr and short __name__/__qualname__, so mocked defaults and annotations render like the real objects Also: get_type_name now prefers __qualname__ for non-typing objects, which renders mocked annotations like real classes and fixes real builds rendering builtin-function annotations as `` (now `callable`). Fidelity, verified by building accelerate docs in a venv WITHOUT torch or deepspeed installed (--mock_deps torch,deepspeed) against a real- torch reference build: 46/48 pages byte-identical (modulo the known nondeterministic 0x addresses); the remaining 2 differ only in typing paths rendered from the public import path (torch.nn.Module) instead of the internal one (torch.nn.modules.module.Module). The documented library itself stays really installed (inspect reads its real source); packages that are actually installed are never mocked. `doc-builder preview` supports the flag too. Co-Authored-By: Claude Fable 5 --- README.md | 15 ++ src/doc_builder/autodoc.py | 6 + src/doc_builder/commands/build.py | 12 ++ src/doc_builder/commands/preview.py | 12 ++ src/doc_builder/mock_imports.py | 228 ++++++++++++++++++++++++++++ tests/test_mock_imports.py | 88 +++++++++++ 6 files changed, 361 insertions(+) create mode 100644 src/doc_builder/mock_imports.py create mode 100644 tests/test_mock_imports.py diff --git a/README.md b/README.md index b6ffc0ec..d7e878b8 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,21 @@ A page is reused when its generated MDX (post autodoc and internal-link resoluti `--html_page_cache_write` also stores freshly built pages. Only enable it on trusted builds (the shared GitHub workflows enable it on main-branch builds and keep PR builds read-only), so that untrusted code cannot poison the cache. Cache failures are never fatal: any error degrades to building the affected pages. +### Building without installing heavy dependencies + +doc-builder imports the documented library to read docstrings with `inspect` — necessary because libraries like `transformers` construct docstrings dynamically. Historically that meant installing the library's full dependency tree (torch, GPU wheels, custom containers) just to build documentation. + +`--mock_deps` mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed: + +```bash +pip install accelerate --no-deps # plus its light dependencies +doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test-build --mock_deps torch,deepspeed +``` + +The mocked packages are importable, pass the `importlib.util.find_spec` + `importlib.metadata` availability checks HF libraries use, are subclassable without affecting the real subclass's signature or docstring, behave as pass-through decorators, and render in signatures like the real objects (`repr(torch.float32) == "torch.float32"`). Packages that are actually installed are never mocked, and `doc-builder preview` supports the flag too. + +Fidelity, verified on `accelerate` built without torch installed: 46/48 pages byte-identical to a real-torch build; the two remaining pages differ only in typing paths rendered from the public import path instead of the internal one (e.g. `torch.nn.Module` instead of `torch.nn.modules.module.Module`). + ### Notebook conversion `doc-builder` can also automatically convert some of the documentation guides or tutorials into notebooks. This requires two steps: diff --git a/src/doc_builder/autodoc.py b/src/doc_builder/autodoc.py index 083cc85d..b00085aa 100644 --- a/src/doc_builder/autodoc.py +++ b/src/doc_builder/autodoc.py @@ -93,6 +93,12 @@ def get_type_name(typ): if isinstance(typ, type): # If it's a class, use its name. return getattr(typ, "__qualname__", None) or getattr(typ, "__name__", None) or str(typ) + # Non-classes carrying a name (e.g. mocked classes from `--mock_deps`) render like + # classes; typing constructs (whose reprs are the readable form) are excluded. + if getattr(typ, "__module__", "") != "typing": + qualname = getattr(typ, "__qualname__", None) + if isinstance(qualname, str): + return qualname return str(typ) # otherwise, trust its string representation diff --git a/src/doc_builder/commands/build.py b/src/doc_builder/commands/build.py index eb42581a..f80d1a93 100644 --- a/src/doc_builder/commands/build.py +++ b/src/doc_builder/commands/build.py @@ -23,6 +23,7 @@ from doc_builder import build_doc, update_versions_file from doc_builder.build_cache import PageCache, hash_kit_tree, page_cache_key +from doc_builder.mock_imports import mock_deps from doc_builder.utils import ( get_default_branch_name, get_doc_config, @@ -148,6 +149,9 @@ def stage_kit_routes(kit_folder, tmp_dir, output_path): def build_command(args): + if args.mock_deps: + # must happen before the documented library is imported + mock_deps(args.mock_deps.split(",")) read_doc_config(args.path_to_docs) if args.html: # Error at the beginning if node is not properly installed. @@ -356,6 +360,14 @@ def build_command_parser(subparsers=None): "`hf://buckets/hf-doc-build/doc-build-cache`) or a local directory. Only used with --html. Pages whose " "generated mdx did not change are reused from the cache instead of being prerendered again.", ) + parser.add_argument( + "--mock_deps", + type=str, + default=None, + help="Comma-separated list of heavy dependencies (e.g. `torch,tensorflow`) to mock instead of importing, so " + "the documented library can be introspected without installing them. Packages that are actually installed " + "are never mocked.", + ) parser.add_argument( "--html_page_cache_write", action="store_true", diff --git a/src/doc_builder/commands/preview.py b/src/doc_builder/commands/preview.py index 8c26c922..b3934a3c 100644 --- a/src/doc_builder/commands/preview.py +++ b/src/doc_builder/commands/preview.py @@ -23,6 +23,7 @@ from doc_builder import build_doc from doc_builder.commands.build import check_node_is_available, docs_node_env, run_npm, stage_kit_routes from doc_builder.commands.convert_doc_file import find_root_git +from doc_builder.mock_imports import mock_deps from doc_builder.utils import ( is_watchdog_available, locate_kit_folder, @@ -137,6 +138,9 @@ def start_sveltekit_dev(kit_dir, env): def preview_command(args): + if args.mock_deps: + # must happen before the documented library is imported + mock_deps(args.mock_deps.split(",")) if not is_watchdog_available(): raise ImportError( "Please install `watchdog` to run `doc-builder preview` command.\nYou can do so through pip: `pip install watchdog`" @@ -216,6 +220,14 @@ def preview_command_parser(subparsers=None): action="store_true", help="Whether docs files do NOT have corresponding python module (like HF course & hub docs).", ) + parser.add_argument( + "--mock_deps", + type=str, + default=None, + help="Comma-separated list of heavy dependencies (e.g. `torch,tensorflow`) to mock instead of importing, so " + "the documented library can be introspected without installing them. Packages that are actually installed " + "are never mocked.", + ) if subparsers is not None: parser.set_defaults(func=preview_command) diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py new file mode 100644 index 00000000..b09a5948 --- /dev/null +++ b/src/doc_builder/mock_imports.py @@ -0,0 +1,228 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Mock heavy dependencies (torch, tensorflow, ...) so that a documented library can be +imported — and introspected with `inspect` — without installing them. + +doc-builder needs to import the documented library because docstrings are often built +dynamically (e.g. transformers' `add_start_docstrings` decorators), which rules out +purely static analysis. But importing e.g. `transformers` or `accelerate` requires +torch and friends, which are slow to install and may need custom containers (GPU +wheels, compiled extensions) just to build documentation. + +`mock_deps(["torch", ...])` installs a `sys.meta_path` finder that provides, for each +mocked package and any of its submodules: + +- importable module objects whose attributes are auto-created **mock classes**, so + `from torch import nn` and `class Model(nn.Module)` work; +- pass-through decorator behavior: calling a mock with a single function/class returns + it unchanged, so `@torch.no_grad()`-style decorators don't swallow the docstrings + that doc-builder is trying to read (same heuristic as Sphinx's `autodoc_mock_imports`); +- a `repr` equal to the dotted path (`repr(torch.float32) == "torch.float32"`), so + mocked default values render in signatures exactly like the real ones; +- fake distribution metadata, so availability checks in HF libraries — typically + `importlib.util.find_spec(pkg)` plus `importlib.metadata.metadata(pkg)` — treat the + mocked package as installed instead of routing to dummy objects. + +The documented library itself (and its light dependencies) must still be really +installed: `inspect` reads the real source, docstrings and signatures from it. +""" + +import importlib.abc +import importlib.machinery +import importlib.metadata +import sys + +MOCK_VERSION = "9999.0.0" + + +class _MockObject: + """ + A mock value for a dotted path (e.g. `torch.float32`): attribute access chains, + calls behave as pass-through decorators, `repr` is the dotted path, and using it + as a base class substitutes a plain-`type` base (PEP 560 `__mro_entries__`), so + real subclasses keep a normal metaclass and `inspect.signature` reads their real + `__init__` instead of a mock's. + """ + + def __init__(self, display_name): + object.__setattr__(self, "__display_name__", display_name) + # annotations render via `__qualname__`/`__name__` (e.g. `torch.nn.Module` + # in a signature shows as `Module`, like the real class). Instance attributes + # shadow the class's own `__qualname__` string. + short_name = display_name.rsplit(".", 1)[-1] + object.__setattr__(self, "__name__", short_name) + object.__setattr__(self, "__qualname__", short_name) + + def __repr__(self): + return self.__display_name__ + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _make_mock(f"{self.__display_name__}.{name}") + + def __call__(self, *args, **kwargs): + # pass decorated functions/classes through unchanged so their docstrings and + # signatures stay intact (a mock used as a decorator, e.g. `@torch.no_grad()`) + if len(args) == 1 and not kwargs and (callable(args[0]) or isinstance(args[0], type)): + return args[0] + return _make_mock(f"{self.__display_name__}()") + + def __mro_entries__(self, bases): + return (_make_base_class(self.__display_name__),) + + # keep common import-time expressions working + def __iter__(self): + return iter(()) + + def __contains__(self, item): + return False + + def __index__(self): + return 0 + + def __bool__(self): + return True + + def __eq__(self, other): + return self is other + + def __hash__(self): + return hash(self.__display_name__) + + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return True + + def __ge__(self, other): + return True + + +_mock_cache = {} +_base_class_cache = {} + + +def _make_mock(display_name): + """Creates (and caches) a mock object for a dotted path.""" + if display_name not in _mock_cache: + _mock_cache[display_name] = _MockObject(display_name) + return _mock_cache[display_name] + + +def _make_base_class(display_name): + """ + Creates (and caches) the plain class substituted when a mock is used as a base + class (`class Model(nn.Module)`). It provides permissive `__init__`/`__getattr__` + fallbacks but has metaclass `type`, keeping subclass introspection intact. + """ + if display_name not in _base_class_cache: + _base_class_cache[display_name] = type( + display_name.rsplit(".", 1)[-1], + (object,), + { + "__display_name__": display_name, + "__module__": display_name.rsplit(".", 1)[0] if "." in display_name else display_name, + "__init__": lambda self, *args, **kwargs: None, + "__getattr__": lambda self, name: _make_mock(f"{display_name}.{name}"), + "__init_subclass__": classmethod(lambda cls, **kwargs: None), + "__class_getitem__": classmethod(lambda cls, item: cls), + }, + ) + return _base_class_cache[display_name] + + +class _MockDistribution(importlib.metadata.Distribution): + """Just enough metadata for `importlib.metadata.metadata/version` to succeed.""" + + def __init__(self, name): + self._name = name + + def read_text(self, filename): + if filename == "METADATA": + return f"Metadata-Version: 2.1\nName: {self._name}\nVersion: {MOCK_VERSION}\n" + return None + + def locate_file(self, path): + return None + + +class MockFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """Meta path finder+loader serving mock modules and metadata for mocked packages.""" + + def __init__(self, packages): + self.packages = set(packages) + + def _is_mocked(self, fullname): + root = fullname.split(".", 1)[0] + return root in self.packages + + def find_spec(self, fullname, path=None, target=None): + if not self._is_mocked(fullname): + return None + spec = importlib.machinery.ModuleSpec(fullname, self, is_package=True) + return spec + + def create_module(self, spec): + import types + + module = types.ModuleType(spec.name) + module.__version__ = MOCK_VERSION + module.__path__ = [] + module.__getattr__ = lambda name: _make_mock(f"{spec.name}.{name}") + return module + + def exec_module(self, module): + pass + + def find_distributions(self, context=importlib.metadata.DistributionFinder.Context()): + name = context.name + if name is None: + return [] + # normalize the distribution name the way importlib.metadata does + if name.replace("-", "_").lower() in {p.replace("-", "_").lower() for p in self.packages}: + return [_MockDistribution(name)] + return [] + + +def mock_deps(packages): + """ + Makes `packages` (e.g. `["torch"]`) importable as mocks. Must be called before the + documented library is imported. No-op for packages that are actually installed + (the real package takes precedence since the finder is appended last... but + availability must be consistent, so mocked packages that are really installed are + skipped explicitly). + """ + packages = [p.strip() for p in packages if p.strip()] + to_mock = [] + for package in packages: + try: + real = importlib.util.find_spec(package) + except (ImportError, ValueError): + real = None + if real is not None: + print(f"[mock-deps] `{package}` is actually installed, not mocking it") + continue + to_mock.append(package) + if to_mock: + finder = MockFinder(to_mock) + sys.meta_path.insert(0, finder) + print(f"[mock-deps] mocking imports for: {', '.join(sorted(to_mock))}") + return to_mock diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py new file mode 100644 index 00000000..08de75ad --- /dev/null +++ b/tests/test_mock_imports.py @@ -0,0 +1,88 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import importlib.util +import inspect +import sys +import unittest + +from doc_builder.mock_imports import MOCK_VERSION, MockFinder, mock_deps + +PKG = "doc_builder_test_mocked_pkg" + + +class MockImportsTester(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.finder = MockFinder([PKG]) + sys.meta_path.insert(0, cls.finder) + + @classmethod + def tearDownClass(cls): + sys.meta_path.remove(cls.finder) + for name in list(sys.modules): + if name.startswith(PKG): + del sys.modules[name] + + def test_mocked_module_imports(self): + module = importlib.import_module(PKG) + self.assertEqual(module.__version__, MOCK_VERSION) + submodule = importlib.import_module(f"{PKG}.sub.module") + self.assertIsNotNone(submodule) + + def test_availability_checks_pass(self): + # the two checks HF libraries typically combine + self.assertIsNotNone(importlib.util.find_spec(PKG)) + self.assertIsNotNone(importlib.metadata.metadata(PKG)) + self.assertEqual(importlib.metadata.version(PKG), MOCK_VERSION) + + def test_mock_repr_and_name(self): + module = importlib.import_module(PKG) + mock = module.nn.Module + self.assertEqual(repr(mock), f"{PKG}.nn.Module") + self.assertEqual(mock.__name__, "Module") + self.assertEqual(mock.__qualname__, "Module") + + def test_subclass_keeps_real_signature(self): + module = importlib.import_module(PKG) + + class Model(module.nn.Module): + """A real class inheriting from a mocked base.""" + + def __init__(self, hidden_size: int = 8, dropout: float = 0.1): + pass + + self.assertEqual(list(inspect.signature(Model).parameters), ["hidden_size", "dropout"]) + self.assertEqual(Model.__doc__, "A real class inheriting from a mocked base.") + # metaclass stays `type`, like a normal class + self.assertIs(type(Model), type) + + def test_mock_decorator_passes_through(self): + module = importlib.import_module(PKG) + + @module.no_grad() + def documented(a: int, b: str = "x"): + """Docstring survives mocked decorators.""" + + self.assertEqual(documented.__doc__, "Docstring survives mocked decorators.") + self.assertEqual(list(inspect.signature(documented).parameters), ["a", "b"]) + + def test_installed_packages_are_not_mocked(self): + # `json` is really installed, so it must be skipped + mocked = mock_deps(["json"]) + self.assertEqual(mocked, []) + import json + + self.assertTrue(json.dumps({"a": 1})) From c6bc08cf04f5fbfd62c96ec025a9678a6b977ef0 Mon Sep 17 00:00:00 2001 From: Mishig Date: Sun, 5 Jul 2026 21:54:25 +0200 Subject: [PATCH 2/5] feat: harden --mock_deps against the full transformers autodoc surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated by building all 723 transformers en doc pages with --mock_deps torch,tensorflow,flax,jax,torchvision,torchaudio,timm and only light real deps installed (tokenizers, sentencepiece, ...): zero conversion errors, full en mdx build in ~9s. Spot-check against production pages built with real deps: bert.md and llama.md are byte-identical (similarity 1.0000); trainer.md at 0.9984 with residual diffs attributable to dependency version skew (older local accelerate). Gaps found and fixed by the validation: - PEP 604 unions in runtime-evaluated annotations (`int | torch.Tensor`) now build *real* typing.Union objects (mocks pass typing._type_check because they are callable), so tools unwrapping Optional/Union via typing.get_origin/get_args — like transformers' auto_docstring — treat them exactly like real annotations. - mock modules are a ModuleType subclass carrying the same operator surface: `from torch import Tensor` on a mocked package materializes `torch.Tensor` as a module via _handle_fromlist, which then appears in unions/calls/base classes. - mock instances expose `__module__` (the mocked path) so `__module__.__qualname__` renderers show `torch.Tensor`, not doc_builder.mock_imports. - mock base classes gained class-level attribute fallback via a metaclass (real subclasses may rely on inherited attributes like nn.Module.forward) deriving from ABCMeta to stay combinable with real ABC bases; it defines no __call__, keeping subclass signatures intact. - find_object_in_package: when a submodule is importable but not bound as an attribute of its (lazy) parent, resolve it from sys.modules and heal the binding — fixes transformers' fusion_mapping-style pages, independent of mocking. Co-Authored-By: Claude Fable 5 --- src/doc_builder/autodoc.py | 11 ++++ src/doc_builder/mock_imports.py | 99 ++++++++++++++++++++++++++++++--- tests/test_mock_imports.py | 30 +++++++++- 3 files changed, 130 insertions(+), 10 deletions(-) diff --git a/src/doc_builder/autodoc.py b/src/doc_builder/autodoc.py index b00085aa..323b1b32 100644 --- a/src/doc_builder/autodoc.py +++ b/src/doc_builder/autodoc.py @@ -16,6 +16,7 @@ import inspect import json import re +import sys from functools import lru_cache from .convert_md_to_mdx import convert_md_docstring_to_mdx @@ -44,6 +45,16 @@ def find_object_in_package(object_name, package): full_module_path = f"{module.__name__}.{split}" importlib.import_module(full_module_path) submodule = getattr(module, split, None) + if submodule is None: + # a submodule can be importable without being bound as an + # attribute of its parent (e.g. lazy-module parents whose + # attribute binding was skipped because the submodule was + # already in sys.modules when first accessed) + submodule = sys.modules.get(full_module_path) + if submodule is not None: + # heal the missing binding so later attribute walks + # (e.g. get_shortest_path) resolve too + setattr(module, split, submodule) except ImportError: return None # Return None if the module cannot be found or imported module = submodule diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py index b09a5948..5c25ad6d 100644 --- a/src/doc_builder/mock_imports.py +++ b/src/doc_builder/mock_imports.py @@ -40,10 +40,13 @@ installed: `inspect` reads the real source, docstrings and signatures from it. """ +import abc import importlib.abc import importlib.machinery import importlib.metadata import sys +import types +import typing MOCK_VERSION = "9999.0.0" @@ -65,6 +68,10 @@ def __init__(self, display_name): short_name = display_name.rsplit(".", 1)[-1] object.__setattr__(self, "__name__", short_name) object.__setattr__(self, "__qualname__", short_name) + # tools rendering type names as `__module__.__qualname__` (e.g. transformers' + # auto_docstring) must see the mocked path, not this module's + parent = display_name.rsplit(".", 1)[0] if "." in display_name else display_name + object.__setattr__(self, "__module__", parent) def __repr__(self): return self.__display_name__ @@ -84,6 +91,26 @@ def __call__(self, *args, **kwargs): def __mro_entries__(self, bases): return (_make_base_class(self.__display_name__),) + # PEP 604 unions in runtime-evaluated annotations (`torch.Tensor | None`): build a + # *real* typing.Union (mocks pass `typing._type_check` because they are callable), + # so tools that unwrap Optional/Union via typing.get_origin/get_args — like + # transformers' auto_docstring — treat it exactly like the real annotation + def __or__(self, other): + try: + return typing.Union[self, other] # noqa: UP007 + except TypeError: + return _make_mock(f"typing.Union[{self.__display_name__}, {_annotation_name(other)}]") + + def __ror__(self, other): + try: + return typing.Union[other, self] # noqa: UP007 + except TypeError: + return _make_mock(f"typing.Union[{_annotation_name(other)}, {self.__display_name__}]") + + def __getitem__(self, item): + # subscripted generics on mocks (`torch.Tensor[int]`-style, rare) + return self + # keep common import-time expressions working def __iter__(self): return iter(()) @@ -116,6 +143,18 @@ def __ge__(self, other): return True +def _annotation_name(obj): + """Renders `obj` the way `typing` renders it inside a union.""" + display_name = getattr(obj, "__display_name__", None) + if display_name is not None: + return display_name + if isinstance(obj, type): + module = getattr(obj, "__module__", "") + qualname = getattr(obj, "__qualname__", str(obj)) + return qualname if module in ("builtins", "") else f"{module}.{qualname}" + return str(obj) + + _mock_cache = {} _base_class_cache = {} @@ -127,14 +166,31 @@ def _make_mock(display_name): return _mock_cache[display_name] +class _MockBaseMeta(abc.ABCMeta): + """ + Metaclass for mock base classes providing *class-level* attribute fallback: real + subclasses may rely on attributes the real base would provide (e.g. a model class + without its own `forward` inherits `torch.nn.Module.forward`). It intentionally + does NOT define `__call__`, so `inspect.signature` on subclasses still reads their + real `__init__` (a metaclass `__call__` would shadow it). Deriving from `ABCMeta` + keeps mock bases combinable with real ABC bases without metaclass conflicts. + """ + + def __getattr__(cls, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + base_display_name = getattr(cls, "__display_name__", cls.__name__) + return _make_mock(f"{base_display_name}.{name}") + + def _make_base_class(display_name): """ - Creates (and caches) the plain class substituted when a mock is used as a base - class (`class Model(nn.Module)`). It provides permissive `__init__`/`__getattr__` - fallbacks but has metaclass `type`, keeping subclass introspection intact. + Creates (and caches) the class substituted when a mock is used as a base class + (`class Model(nn.Module)`). It provides permissive `__init__` and instance/class + attribute fallbacks while keeping subclass introspection intact. """ if display_name not in _base_class_cache: - _base_class_cache[display_name] = type( + _base_class_cache[display_name] = _MockBaseMeta( display_name.rsplit(".", 1)[-1], (object,), { @@ -149,6 +205,36 @@ class (`class Model(nn.Module)`). It provides permissive `__init__`/`__getattr__ return _base_class_cache[display_name] +class _MockModule(types.ModuleType): + """ + Mock module supporting the same expression surface as `_MockObject`: once code runs + `import torch.X`, the import system binds a real module object as the `X` attribute + of `torch` (shadowing the mock), and that module may then appear in runtime + expressions like PEP 604 annotations (`torch.Tensor | None` when `torch.Tensor` was + imported as a module path) or be called/subclassed. + """ + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _make_mock(f"{self.__name__}.{name}") + + def __call__(self, *args, **kwargs): + return _make_mock(f"{self.__name__}")(*args, **kwargs) + + def __or__(self, other): + return _make_mock(f"{self.__name__}").__or__(other) + + def __ror__(self, other): + return _make_mock(f"{self.__name__}").__ror__(other) + + def __mro_entries__(self, bases): + return (_make_base_class(self.__name__),) + + def __getitem__(self, item): + return self + + class _MockDistribution(importlib.metadata.Distribution): """Just enough metadata for `importlib.metadata.metadata/version` to succeed.""" @@ -181,12 +267,9 @@ def find_spec(self, fullname, path=None, target=None): return spec def create_module(self, spec): - import types - - module = types.ModuleType(spec.name) + module = _MockModule(spec.name) module.__version__ = MOCK_VERSION module.__path__ = [] - module.__getattr__ = lambda name: _make_mock(f"{spec.name}.{name}") return module def exec_module(self, module): diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py index 08de75ad..063225ee 100644 --- a/tests/test_mock_imports.py +++ b/tests/test_mock_imports.py @@ -66,8 +66,9 @@ def __init__(self, hidden_size: int = 8, dropout: float = 0.1): self.assertEqual(list(inspect.signature(Model).parameters), ["hidden_size", "dropout"]) self.assertEqual(Model.__doc__, "A real class inheriting from a mocked base.") - # metaclass stays `type`, like a normal class - self.assertIs(type(Model), type) + # the metaclass must not define `__call__`: it would shadow the subclass's + # real `__init__` under `inspect.signature` + self.assertNotIn("__call__", type(Model).__dict__) def test_mock_decorator_passes_through(self): module = importlib.import_module(PKG) @@ -79,6 +80,31 @@ def documented(a: int, b: str = "x"): self.assertEqual(documented.__doc__, "Docstring survives mocked decorators.") self.assertEqual(list(inspect.signature(documented).parameters), ["a", "b"]) + def test_pep604_unions_are_real_typing_unions(self): + import typing + + module = importlib.import_module(PKG) + union = module.Tensor | None + # a real typing.Union, so tools unwrapping Optional via get_origin/get_args + # (e.g. transformers' auto_docstring) treat it like the real annotation + self.assertIs(typing.get_origin(union), typing.Union) + args = typing.get_args(union) + self.assertIs(args[1], type(None)) + self.assertEqual(getattr(args[0], "__module__", None), PKG) + self.assertEqual(getattr(args[0], "__qualname__", None), "Tensor") + # reflected form with a real type on the left + union = int | module.Tensor + self.assertIs(typing.get_origin(union), typing.Union) + + def test_subclass_inherits_mock_class_attributes(self): + module = importlib.import_module(PKG) + + class Model(module.nn.Module): + pass + + # class-level fallback: the real base (e.g. nn.Module) would provide `forward` + self.assertIsNotNone(getattr(Model, "forward", None)) + def test_installed_packages_are_not_mocked(self): # `json` is really installed, so it must be skipped mocked = mock_deps(["json"]) From d4a45f6755ffde1e12c1d328a1ed48f2abc2f020 Mon Sep 17 00:00:00 2001 From: Mishig Date: Sun, 5 Jul 2026 23:54:46 +0200 Subject: [PATCH 3/5] Add per-library mock-deps registry, applied automatically Each library gets a src/doc_builder/mock_deps/.txt listing the heavy dependencies to mock when building its docs; build/preview apply it automatically, so workflows no longer need --mock_deps (kept as an additive extra). Registry validated by building the docs of 17 HF libraries end to end with only light real deps installed. Also hardens the mocks for cases those builds surfaced: mocked module __version__ compares against tuples like torch's TorchVersion, len() of a mock returns 0, and class-attribute fallback on mocked bases is restricted to a whitelist (forward/call) so hasattr probes on subclasses stay accurate. Co-Authored-By: Claude Fable 5 --- README.md | 7 ++- setup.py | 2 + src/doc_builder/commands/build.py | 13 ++-- src/doc_builder/commands/preview.py | 13 ++-- src/doc_builder/mock_deps/accelerate.txt | 5 ++ src/doc_builder/mock_deps/autotrain.txt | 4 ++ src/doc_builder/mock_deps/bitsandbytes.txt | 3 + src/doc_builder/mock_deps/datasets.txt | 2 + src/doc_builder/mock_deps/diffusers.txt | 3 + src/doc_builder/mock_deps/evaluate.txt | 4 ++ src/doc_builder/mock_deps/huggingface_hub.txt | 3 + src/doc_builder/mock_deps/lighteval.txt | 7 +++ src/doc_builder/mock_deps/openenv.txt | 9 +++ src/doc_builder/mock_deps/peft.txt | 3 + src/doc_builder/mock_deps/safetensors.txt | 6 ++ src/doc_builder/mock_deps/setfit.txt | 13 ++++ src/doc_builder/mock_deps/timm.txt | 4 ++ src/doc_builder/mock_deps/trackio.txt | 4 ++ src/doc_builder/mock_deps/transformers.txt | 10 +++ src/doc_builder/mock_deps/trl.txt | 3 + src/doc_builder/mock_imports.py | 63 ++++++++++++++++++- tests/test_mock_imports.py | 13 ++++ 22 files changed, 182 insertions(+), 12 deletions(-) create mode 100644 src/doc_builder/mock_deps/accelerate.txt create mode 100644 src/doc_builder/mock_deps/autotrain.txt create mode 100644 src/doc_builder/mock_deps/bitsandbytes.txt create mode 100644 src/doc_builder/mock_deps/datasets.txt create mode 100644 src/doc_builder/mock_deps/diffusers.txt create mode 100644 src/doc_builder/mock_deps/evaluate.txt create mode 100644 src/doc_builder/mock_deps/huggingface_hub.txt create mode 100644 src/doc_builder/mock_deps/lighteval.txt create mode 100644 src/doc_builder/mock_deps/openenv.txt create mode 100644 src/doc_builder/mock_deps/peft.txt create mode 100644 src/doc_builder/mock_deps/safetensors.txt create mode 100644 src/doc_builder/mock_deps/setfit.txt create mode 100644 src/doc_builder/mock_deps/timm.txt create mode 100644 src/doc_builder/mock_deps/trackio.txt create mode 100644 src/doc_builder/mock_deps/transformers.txt create mode 100644 src/doc_builder/mock_deps/trl.txt diff --git a/README.md b/README.md index d7e878b8..221f8d72 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,16 @@ A page is reused when its generated MDX (post autodoc and internal-link resoluti doc-builder imports the documented library to read docstrings with `inspect` — necessary because libraries like `transformers` construct docstrings dynamically. Historically that meant installing the library's full dependency tree (torch, GPU wheels, custom containers) just to build documentation. -`--mock_deps` mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed: +doc-builder mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed. The list of mockable dependencies per library lives in `src/doc_builder/mock_deps/.txt` and is applied automatically: ```bash pip install accelerate --no-deps # plus its light dependencies -doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test-build --mock_deps torch,deepspeed +doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test-build +# torch and deepspeed are mocked automatically (src/doc_builder/mock_deps/accelerate.txt) ``` +For a library without a registry entry (or to experiment), `--mock_deps torch,...` adds extra mocks on top of the registry. + The mocked packages are importable, pass the `importlib.util.find_spec` + `importlib.metadata` availability checks HF libraries use, are subclassable without affecting the real subclass's signature or docstring, behave as pass-through decorators, and render in signatures like the real objects (`repr(torch.float32) == "torch.float32"`). Packages that are actually installed are never mocked, and `doc-builder preview` supports the flag too. Fidelity, verified on `accelerate` built without torch installed: 46/48 pages byte-identical to a real-torch build; the two remaining pages differ only in typing paths rendered from the public import path instead of the internal one (e.g. `torch.nn.Module` instead of `torch.nn.modules.module.Module`). diff --git a/setup.py b/setup.py index e468a5a9..61a62151 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,8 @@ keywords="doc documentation doc-builder huggingface hugging face", url="https://github.com/huggingface/doc-builder", package_dir={"": "src"}, + package_data={"doc_builder": ["mock_deps/*.txt"]}, + include_package_data=True, packages=find_packages("src"), extras_require=extras, install_requires=install_requires, diff --git a/src/doc_builder/commands/build.py b/src/doc_builder/commands/build.py index f80d1a93..362961d3 100644 --- a/src/doc_builder/commands/build.py +++ b/src/doc_builder/commands/build.py @@ -23,7 +23,7 @@ from doc_builder import build_doc, update_versions_file from doc_builder.build_cache import PageCache, hash_kit_tree, page_cache_key -from doc_builder.mock_imports import mock_deps +from doc_builder.mock_imports import get_registry_mock_deps, mock_deps from doc_builder.utils import ( get_default_branch_name, get_doc_config, @@ -149,9 +149,14 @@ def stage_kit_routes(kit_folder, tmp_dir, output_path): def build_command(args): - if args.mock_deps: - # must happen before the documented library is imported - mock_deps(args.mock_deps.split(",")) + # heavy deps that can be mocked come from the per-library registry + # (doc_builder/mock_deps/.txt); --mock_deps adds extra ones. + # Must happen before the documented library is imported; packages that are + # actually installed are never mocked. + registry_deps = get_registry_mock_deps(args.library_name) + extra_deps = args.mock_deps.split(",") if args.mock_deps else [] + if registry_deps or extra_deps: + mock_deps(registry_deps + extra_deps) read_doc_config(args.path_to_docs) if args.html: # Error at the beginning if node is not properly installed. diff --git a/src/doc_builder/commands/preview.py b/src/doc_builder/commands/preview.py index b3934a3c..0c5c66e2 100644 --- a/src/doc_builder/commands/preview.py +++ b/src/doc_builder/commands/preview.py @@ -23,7 +23,7 @@ from doc_builder import build_doc from doc_builder.commands.build import check_node_is_available, docs_node_env, run_npm, stage_kit_routes from doc_builder.commands.convert_doc_file import find_root_git -from doc_builder.mock_imports import mock_deps +from doc_builder.mock_imports import get_registry_mock_deps, mock_deps from doc_builder.utils import ( is_watchdog_available, locate_kit_folder, @@ -138,9 +138,14 @@ def start_sveltekit_dev(kit_dir, env): def preview_command(args): - if args.mock_deps: - # must happen before the documented library is imported - mock_deps(args.mock_deps.split(",")) + # heavy deps that can be mocked come from the per-library registry + # (doc_builder/mock_deps/.txt); --mock_deps adds extra ones. + # Must happen before the documented library is imported; packages that are + # actually installed are never mocked. + registry_deps = get_registry_mock_deps(args.library_name) + extra_deps = args.mock_deps.split(",") if args.mock_deps else [] + if registry_deps or extra_deps: + mock_deps(registry_deps + extra_deps) if not is_watchdog_available(): raise ImportError( "Please install `watchdog` to run `doc-builder preview` command.\nYou can do so through pip: `pip install watchdog`" diff --git a/src/doc_builder/mock_deps/accelerate.txt b/src/doc_builder/mock_deps/accelerate.txt new file mode 100644 index 00000000..abfdefde --- /dev/null +++ b/src/doc_builder/mock_deps/accelerate.txt @@ -0,0 +1,5 @@ +# Heavy dependencies mocked when building accelerate docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +deepspeed +torchvision diff --git a/src/doc_builder/mock_deps/autotrain.txt b/src/doc_builder/mock_deps/autotrain.txt new file mode 100644 index 00000000..f23c8c26 --- /dev/null +++ b/src/doc_builder/mock_deps/autotrain.txt @@ -0,0 +1,4 @@ +# Heavy dependencies mocked when building autotrain docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +loguru diff --git a/src/doc_builder/mock_deps/bitsandbytes.txt b/src/doc_builder/mock_deps/bitsandbytes.txt new file mode 100644 index 00000000..3e0cd29a --- /dev/null +++ b/src/doc_builder/mock_deps/bitsandbytes.txt @@ -0,0 +1,3 @@ +# Heavy dependencies mocked when building bitsandbytes docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch diff --git a/src/doc_builder/mock_deps/datasets.txt b/src/doc_builder/mock_deps/datasets.txt new file mode 100644 index 00000000..cb90d499 --- /dev/null +++ b/src/doc_builder/mock_deps/datasets.txt @@ -0,0 +1,2 @@ +# datasets docs build without mocking: its framework integrations are optional +# and its required deps (pyarrow, pandas, ...) must be really installed. diff --git a/src/doc_builder/mock_deps/diffusers.txt b/src/doc_builder/mock_deps/diffusers.txt new file mode 100644 index 00000000..0aabb075 --- /dev/null +++ b/src/doc_builder/mock_deps/diffusers.txt @@ -0,0 +1,3 @@ +# Heavy dependencies mocked when building diffusers docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch diff --git a/src/doc_builder/mock_deps/evaluate.txt b/src/doc_builder/mock_deps/evaluate.txt new file mode 100644 index 00000000..ab1ceb43 --- /dev/null +++ b/src/doc_builder/mock_deps/evaluate.txt @@ -0,0 +1,4 @@ +# Heavy dependencies mocked when building evaluate docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +matplotlib diff --git a/src/doc_builder/mock_deps/huggingface_hub.txt b/src/doc_builder/mock_deps/huggingface_hub.txt new file mode 100644 index 00000000..c7c85c3e --- /dev/null +++ b/src/doc_builder/mock_deps/huggingface_hub.txt @@ -0,0 +1,3 @@ +# Heavy dependencies mocked when building huggingface_hub docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch diff --git a/src/doc_builder/mock_deps/lighteval.txt b/src/doc_builder/mock_deps/lighteval.txt new file mode 100644 index 00000000..10ffdd12 --- /dev/null +++ b/src/doc_builder/mock_deps/lighteval.txt @@ -0,0 +1,7 @@ +# Heavy dependencies mocked when building lighteval docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +vllm +matplotlib +ray +more_itertools diff --git a/src/doc_builder/mock_deps/openenv.txt b/src/doc_builder/mock_deps/openenv.txt new file mode 100644 index 00000000..f8d4f4e8 --- /dev/null +++ b/src/doc_builder/mock_deps/openenv.txt @@ -0,0 +1,9 @@ +# Heavy dependencies mocked when building openenv docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +tomli_w +fastapi +fastmcp +mcp +openai +gradio +websockets diff --git a/src/doc_builder/mock_deps/peft.txt b/src/doc_builder/mock_deps/peft.txt new file mode 100644 index 00000000..84d13df1 --- /dev/null +++ b/src/doc_builder/mock_deps/peft.txt @@ -0,0 +1,3 @@ +# Heavy dependencies mocked when building peft docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch diff --git a/src/doc_builder/mock_deps/safetensors.txt b/src/doc_builder/mock_deps/safetensors.txt new file mode 100644 index 00000000..cc05ec5f --- /dev/null +++ b/src/doc_builder/mock_deps/safetensors.txt @@ -0,0 +1,6 @@ +# Heavy dependencies mocked when building safetensors docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +paddle +torch +tensorflow +jax diff --git a/src/doc_builder/mock_deps/setfit.txt b/src/doc_builder/mock_deps/setfit.txt new file mode 100644 index 00000000..5f41ea21 --- /dev/null +++ b/src/doc_builder/mock_deps/setfit.txt @@ -0,0 +1,13 @@ +# Heavy dependencies mocked when building setfit docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +sentence_transformers +joblib +sklearn +matplotlib +tensorflow +flax +jax +torchvision +torchaudio +scipy diff --git a/src/doc_builder/mock_deps/timm.txt b/src/doc_builder/mock_deps/timm.txt new file mode 100644 index 00000000..1e1b87a2 --- /dev/null +++ b/src/doc_builder/mock_deps/timm.txt @@ -0,0 +1,4 @@ +# Heavy dependencies mocked when building timm docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +torchvision diff --git a/src/doc_builder/mock_deps/trackio.txt b/src/doc_builder/mock_deps/trackio.txt new file mode 100644 index 00000000..be6281b5 --- /dev/null +++ b/src/doc_builder/mock_deps/trackio.txt @@ -0,0 +1,4 @@ +# Heavy dependencies mocked when building trackio docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +matplotlib diff --git a/src/doc_builder/mock_deps/transformers.txt b/src/doc_builder/mock_deps/transformers.txt new file mode 100644 index 00000000..19b9bf36 --- /dev/null +++ b/src/doc_builder/mock_deps/transformers.txt @@ -0,0 +1,10 @@ +# Heavy dependencies mocked when building transformers docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +tensorflow +flax +jax +torchvision +torchaudio +timm +deepspeed diff --git a/src/doc_builder/mock_deps/trl.txt b/src/doc_builder/mock_deps/trl.txt new file mode 100644 index 00000000..5137969a --- /dev/null +++ b/src/doc_builder/mock_deps/trl.txt @@ -0,0 +1,3 @@ +# Heavy dependencies mocked when building trl docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py index 5c25ad6d..8b324858 100644 --- a/src/doc_builder/mock_imports.py +++ b/src/doc_builder/mock_imports.py @@ -47,10 +47,42 @@ import sys import types import typing +from pathlib import Path MOCK_VERSION = "9999.0.0" +class _MockVersionString(str): + """ + Version string comparable against tuples, like torch's `TorchVersion` + (libraries write e.g. `torch.__version__ >= (2, 6)`). + """ + + def _as_tuple(self): + return tuple(int(p) for p in self.split(".") if p.isdigit()) + + def _coerce(self, other): + if isinstance(other, tuple): + return tuple(int(p) if not isinstance(p, int) else p for p in other) + return None + + def __ge__(self, other): + coerced = self._coerce(other) + return self._as_tuple() >= coerced if coerced is not None else str.__ge__(self, other) + + def __gt__(self, other): + coerced = self._coerce(other) + return self._as_tuple() > coerced if coerced is not None else str.__gt__(self, other) + + def __le__(self, other): + coerced = self._coerce(other) + return self._as_tuple() <= coerced if coerced is not None else str.__le__(self, other) + + def __lt__(self, other): + coerced = self._coerce(other) + return self._as_tuple() < coerced if coerced is not None else str.__lt__(self, other) + + class _MockObject: """ A mock value for a dotted path (e.g. `torch.float32`): attribute access chains, @@ -112,6 +144,9 @@ def __getitem__(self, item): return self # keep common import-time expressions working + def __len__(self): + return 0 + def __iter__(self): return iter(()) @@ -176,8 +211,15 @@ class _MockBaseMeta(abc.ABCMeta): keeps mock bases combinable with real ABC bases without metaclass conflicts. """ + # only fall back for methods commonly inherited from framework bases and + # documented as such (e.g. `[[autodoc]] SomeModel - forward` where the model + # doesn't override nn.Module.forward). A blanket fallback would make + # `hasattr(cls, anything)` true and break libraries' own attribute checks + # (e.g. peft's prefix-registration validation). + _FALLBACK_ATTRIBUTES = {"forward", "call"} + def __getattr__(cls, name): - if name.startswith("__") and name.endswith("__"): + if name not in _MockBaseMeta._FALLBACK_ATTRIBUTES: raise AttributeError(name) base_display_name = getattr(cls, "__display_name__", cls.__name__) return _make_mock(f"{base_display_name}.{name}") @@ -268,7 +310,7 @@ def find_spec(self, fullname, path=None, target=None): def create_module(self, spec): module = _MockModule(spec.name) - module.__version__ = MOCK_VERSION + module.__version__ = _MockVersionString(MOCK_VERSION) module.__path__ = [] return module @@ -285,6 +327,23 @@ def find_distributions(self, context=importlib.metadata.DistributionFinder.Conte return [] +def get_registry_mock_deps(library_name): + """ + Returns the registered list of mockable heavy dependencies for a documented + library, from `doc_builder/mock_deps/.txt` (one import name per + line, `#` comments). Empty when the library has no registry entry. + """ + registry_file = Path(__file__).parent / "mock_deps" / f"{library_name}.txt" + if not registry_file.is_file(): + return [] + deps = [] + for line in registry_file.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line: + deps.append(line) + return deps + + def mock_deps(packages): """ Makes `packages` (e.g. `["torch"]`) importable as mocks. Must be called before the diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py index 063225ee..9f6abd3c 100644 --- a/tests/test_mock_imports.py +++ b/tests/test_mock_imports.py @@ -105,6 +105,19 @@ class Model(module.nn.Module): # class-level fallback: the real base (e.g. nn.Module) would provide `forward` self.assertIsNotNone(getattr(Model, "forward", None)) + def test_registry_mock_deps(self): + from doc_builder.mock_imports import get_registry_mock_deps + + deps = get_registry_mock_deps("transformers") + self.assertIn("torch", deps) + self.assertIn("tensorflow", deps) + # comments are stripped + self.assertTrue(all(not d.startswith("#") for d in deps)) + # unknown library -> empty + self.assertEqual(get_registry_mock_deps("not-a-real-library"), []) + # validated as needing no mocks -> empty but present + self.assertEqual(get_registry_mock_deps("datasets"), []) + def test_installed_packages_are_not_mocked(self): # `json` is really installed, so it must be skipped mocked = mock_deps(["json"]) From fd872714766288ea0279f94f390873d007c423fe Mon Sep 17 00:00:00 2001 From: Mishig Date: Mon, 6 Jul 2026 08:55:02 +0200 Subject: [PATCH 4/5] Extend the mock-deps registry to the optimum family Registry keys are the doc-builder library names, so namespace subpackages get dotted files (optimum.intel.txt, optimum.neuron.txt, ...) and optimum-onnx shares optimum.txt since its docs build under the library name "optimum". MockFinder now matches registered names by prefix instead of top-level root, so a registered name can itself be a dotted submodule (mocking optimum.onnxruntime while the real optimum stays importable). Validated end to end: optimum (16 pages), optimum-onnx (23), optimum.intel (9), optimum.executorch (5), optimum.neuron (53, with the workflow's notebook-to-mdx pre-steps), optimum.tpu (19), optimum.habana (21, needs its version-pinned real deps plus a habana-torch-plugin distribution visible to `pip list` because of an import-time subprocess version check). Co-Authored-By: Claude Fable 5 --- README.md | 2 ++ .../mock_deps/optimum.executorch.txt | 1 + src/doc_builder/mock_deps/optimum.habana.txt | 7 +++++++ src/doc_builder/mock_deps/optimum.intel.txt | 4 ++++ src/doc_builder/mock_deps/optimum.neuron.txt | 9 +++++++++ src/doc_builder/mock_deps/optimum.tpu.txt | 1 + src/doc_builder/mock_deps/optimum.txt | 7 +++++++ src/doc_builder/mock_imports.py | 6 ++++-- tests/test_mock_imports.py | 18 ++++++++++++++++++ 9 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 src/doc_builder/mock_deps/optimum.executorch.txt create mode 100644 src/doc_builder/mock_deps/optimum.habana.txt create mode 100644 src/doc_builder/mock_deps/optimum.intel.txt create mode 100644 src/doc_builder/mock_deps/optimum.neuron.txt create mode 100644 src/doc_builder/mock_deps/optimum.tpu.txt create mode 100644 src/doc_builder/mock_deps/optimum.txt diff --git a/README.md b/README.md index 221f8d72..569e29b8 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,8 @@ doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test For a library without a registry entry (or to experiment), `--mock_deps torch,...` adds extra mocks on top of the registry. +The registry key is the doc-builder *library name* as passed to `doc-builder build` — for namespace subpackages that is the dotted module name (e.g. `mock_deps/optimum.intel.txt`), and libraries that build under a shared name share a file (optimum-onnx builds as `optimum`). A registered name may itself be a dotted submodule (e.g. `optimum.onnxruntime`), which mocks a missing sibling inside an installed namespace package. + The mocked packages are importable, pass the `importlib.util.find_spec` + `importlib.metadata` availability checks HF libraries use, are subclassable without affecting the real subclass's signature or docstring, behave as pass-through decorators, and render in signatures like the real objects (`repr(torch.float32) == "torch.float32"`). Packages that are actually installed are never mocked, and `doc-builder preview` supports the flag too. Fidelity, verified on `accelerate` built without torch installed: 46/48 pages byte-identical to a real-torch build; the two remaining pages differ only in typing paths rendered from the public import path instead of the internal one (e.g. `torch.nn.Module` instead of `torch.nn.modules.module.Module`). diff --git a/src/doc_builder/mock_deps/optimum.executorch.txt b/src/doc_builder/mock_deps/optimum.executorch.txt new file mode 100644 index 00000000..1678b500 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.executorch.txt @@ -0,0 +1 @@ +# optimum-executorch docs build without mocks (validated with only the library installed). diff --git a/src/doc_builder/mock_deps/optimum.habana.txt b/src/doc_builder/mock_deps/optimum.habana.txt new file mode 100644 index 00000000..c8dbbe23 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.habana.txt @@ -0,0 +1,7 @@ +# Heavy dependencies mocked when building optimum.habana docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +# NOTE: also requires version-pinned real deps (transformers, accelerate, diffusers, +# sentence-transformers) and a `habana-torch-plugin` distribution visible to `pip list`. +torch +sklearn +scipy diff --git a/src/doc_builder/mock_deps/optimum.intel.txt b/src/doc_builder/mock_deps/optimum.intel.txt new file mode 100644 index 00000000..e98e836f --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.intel.txt @@ -0,0 +1,4 @@ +# Heavy dependencies mocked when building optimum.intel docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +openvino diff --git a/src/doc_builder/mock_deps/optimum.neuron.txt b/src/doc_builder/mock_deps/optimum.neuron.txt new file mode 100644 index 00000000..598800de --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.neuron.txt @@ -0,0 +1,9 @@ +# Heavy dependencies mocked when building optimum.neuron docs (see mock_imports.py). +# Packages that are actually installed are never mocked. +torch +neuronx_distributed +torch_xla +torch_neuronx +libneuronxla +neuronxcc +peft diff --git a/src/doc_builder/mock_deps/optimum.tpu.txt b/src/doc_builder/mock_deps/optimum.tpu.txt new file mode 100644 index 00000000..d7541ffa --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.tpu.txt @@ -0,0 +1 @@ +# optimum-tpu docs build without mocks (validated with only the library installed). diff --git a/src/doc_builder/mock_deps/optimum.txt b/src/doc_builder/mock_deps/optimum.txt new file mode 100644 index 00000000..be977b53 --- /dev/null +++ b/src/doc_builder/mock_deps/optimum.txt @@ -0,0 +1,7 @@ +# Heavy dependencies mocked when building optimum docs (see mock_imports.py). +# Shared by optimum-onnx, whose docs also build under the library name `optimum` +# (only --repo_name differs): onnxruntime/onnx belong to that build. +# Packages that are actually installed are never mocked. +torch +onnxruntime +onnx diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py index 8b324858..0b5a202b 100644 --- a/src/doc_builder/mock_imports.py +++ b/src/doc_builder/mock_imports.py @@ -299,8 +299,10 @@ def __init__(self, packages): self.packages = set(packages) def _is_mocked(self, fullname): - root = fullname.split(".", 1)[0] - return root in self.packages + # prefix matching so a registered name can be a submodule of a namespace + # package (e.g. mock `optimum.onnxruntime` while the real `optimum` and + # `optimum.intel` stay importable) + return any(fullname == package or fullname.startswith(package + ".") for package in self.packages) def find_spec(self, fullname, path=None, target=None): if not self._is_mocked(fullname): diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py index 9f6abd3c..bd075e3d 100644 --- a/tests/test_mock_imports.py +++ b/tests/test_mock_imports.py @@ -105,6 +105,24 @@ class Model(module.nn.Module): # class-level fallback: the real base (e.g. nn.Module) would provide `forward` self.assertIsNotNone(getattr(Model, "forward", None)) + def test_dotted_submodule_mock(self): + # a registered dotted name mocks a missing submodule of a real (namespace) + # package, like `optimum.onnxruntime` when only `optimum` is installed + finder = MockFinder(["unittest.doc_builder_missing_sibling"]) + sys.meta_path.insert(0, finder) + try: + module = importlib.import_module("unittest.doc_builder_missing_sibling.sub") + self.assertEqual(module.__name__, "unittest.doc_builder_missing_sibling.sub") + import unittest as real_unittest + + # the real parent package is untouched + self.assertIs(real_unittest.TestCase, unittest.TestCase) + finally: + sys.meta_path.remove(finder) + for name in list(sys.modules): + if name.startswith("unittest.doc_builder_missing_sibling"): + del sys.modules[name] + def test_registry_mock_deps(self): from doc_builder.mock_imports import get_registry_mock_deps From fbf10a57568d604adf2028c8c9d4f7f92bb40355 Mon Sep 17 00:00:00 2001 From: Mishig Date: Mon, 6 Jul 2026 09:42:47 +0200 Subject: [PATCH 5/5] Registry-driven light installs; drop custom_container; uv-only workflows The mock-deps registry now records both sides of a light doc build: bare lines are mocked, real: lines are installed for real, and nodeps: lines are installed with --no-deps (their own dependency trees pull in heavy packages). The new `doc-builder light-install ` command installs the real deps via uv, pinning bare names to the documented library's own declared version ranges so registry files never chase upstream pins (e.g. transformers' tokenizers range); `--check` reports whether a registry entry exists (exit 3 when not). The shared build workflows install the package with --no-deps and light-install when the package has a registry entry, falling back to the old full [dev] install otherwise. The custom_container input is removed (the light path makes prebuilt-heavy-deps images unnecessary) and every workflow now uses uv exclusively: setup-uv replaces the pip bootstrap, and hf sync runs through uvx. The accelerate integration test exercises the light path end to end. Mock hardening the validation surfaced: issubclass()/isinstance() against a mock answers False instead of raising (scipy>=1.17 probes `issubclass(cls, torch.Tensor)` whenever torch is in sys.modules). Every registry file was validated in a fresh venv containing only doc-builder, the registry's real deps, and the library --no-deps: all 25 builds (18 libraries + optimum family) pass on first iteration. Co-Authored-By: Claude Fable 5 --- .github/workflows/accelerate_doc.yml | 7 +- .../workflows/build_main_documentation.yml | 73 +++------- .github/workflows/build_pr_documentation.yml | 66 +++------ .../delete_old_pr_documentations.yml | 3 +- .github/workflows/upload_pr_documentation.yml | 10 +- README.md | 13 +- src/doc_builder/commands/doc_builder_cli.py | 2 + src/doc_builder/commands/light_install.py | 136 ++++++++++++++++++ src/doc_builder/mock_deps/accelerate.txt | 6 + src/doc_builder/mock_deps/autotrain.txt | 15 ++ src/doc_builder/mock_deps/bitsandbytes.txt | 2 + src/doc_builder/mock_deps/datasets.txt | 13 ++ src/doc_builder/mock_deps/diffusers.txt | 9 ++ src/doc_builder/mock_deps/evaluate.txt | 11 ++ src/doc_builder/mock_deps/huggingface_hub.txt | 7 + src/doc_builder/mock_deps/kernels.txt | 8 ++ src/doc_builder/mock_deps/lerobot.txt | 7 + src/doc_builder/mock_deps/lighteval.txt | 20 +++ src/doc_builder/mock_deps/openenv.txt | 4 + .../mock_deps/optimum.executorch.txt | 5 + src/doc_builder/mock_deps/optimum.habana.txt | 7 + src/doc_builder/mock_deps/optimum.intel.txt | 6 + src/doc_builder/mock_deps/optimum.neuron.txt | 9 ++ src/doc_builder/mock_deps/optimum.tpu.txt | 5 + src/doc_builder/mock_deps/optimum.txt | 6 + src/doc_builder/mock_deps/peft.txt | 9 ++ src/doc_builder/mock_deps/safetensors.txt | 1 + src/doc_builder/mock_deps/timm.txt | 4 + src/doc_builder/mock_deps/tokenizers.txt | 4 + src/doc_builder/mock_deps/trackio.txt | 7 + src/doc_builder/mock_deps/transformers.txt | 14 ++ src/doc_builder/mock_deps/trl.txt | 7 + src/doc_builder/mock_imports.py | 57 ++++++-- tests/test_mock_imports.py | 19 +++ 34 files changed, 452 insertions(+), 120 deletions(-) create mode 100644 src/doc_builder/commands/light_install.py create mode 100644 src/doc_builder/mock_deps/kernels.txt create mode 100644 src/doc_builder/mock_deps/lerobot.txt create mode 100644 src/doc_builder/mock_deps/tokenizers.txt diff --git a/.github/workflows/accelerate_doc.yml b/.github/workflows/accelerate_doc.yml index db1d1c64..62132787 100644 --- a/.github/workflows/accelerate_doc.yml +++ b/.github/workflows/accelerate_doc.yml @@ -38,7 +38,12 @@ jobs: run: uv venv --python 3.10 .venv - name: Install Python dependencies - run: uv pip install --python .venv/bin/python ./doc-builder './accelerate[dev]' + # light install: only accelerate's real (light) doc-build deps; torch & co are + # mocked at build time from the mock-deps registry + run: | + uv pip install --python .venv/bin/python ./doc-builder + uv pip install --python .venv/bin/python --no-deps ./accelerate + ./.venv/bin/doc-builder light-install accelerate - name: Make documentation run: | diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml index e698a8d5..abcc672e 100644 --- a/.github/workflows/build_main_documentation.yml +++ b/.github/workflows/build_main_documentation.yml @@ -46,12 +46,6 @@ on: convert_notebooks: type: boolean description: "Convert notebooks to markdown files before building docs." - # Docker image to use for the build. Set custom_container="huggingface/transformers-doc-builder" if you need - # a complete install. Default to `""` which means CI will run on the runner directly. - custom_container: - type: string - default: "" - description: "Docker image to use for the build." secrets: hf_token: required: true @@ -61,8 +55,6 @@ on: jobs: build_main_documentation: runs-on: ubuntu-22.04 - container: - ${{ inputs.custom_container }} defaults: run: shell: bash @@ -92,28 +84,7 @@ jobs: node-version: '20' cache-dependency-path: "kit/package-lock.json" - - name: Export ROOT_APT_GET ('apt-get' or 'sudo apt-get') - # Use `sudo` only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed) - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "ROOT_APT_GET=sudo apt-get" >> $GITHUB_ENV - else - echo "ROOT_APT_GET=apt-get" >> $GITHUB_ENV - fi - - - name: Export PIP_OR_UV ('pip' or 'uv pip') - # Use `uv` only if running on the base runner. - # When using a container, `pip` has already been used to installed existing deps - # and is therefore quicker to resolve (already have some cached stuff). - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "PIP_OR_UV=uv pip" >> $GITHUB_ENV - else - echo "PIP_OR_UV=pip" >> $GITHUB_ENV - fi + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Set env variables run: | @@ -134,13 +105,11 @@ jobs: # Needed to upload zips to hf.co - name: Install zip - # Use sudo only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed). - run: $ROOT_APT_GET install -y zip + run: sudo apt-get install -y zip - name: Install libgl1 if: inputs.install_libgl1 - run: $ROOT_APT_GET install -y libgl1 + run: sudo apt-get install -y libgl1 - name: Install Rust uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master 2026-03-27 @@ -148,32 +117,33 @@ jobs: with: toolchain: stable - # Use venv in both cases - - name: Install uv - run: | - pip install -U uv - uv venv - - name: Setup environment shell: bash run: | + uv venv source .venv/bin/activate - $PIP_OR_UV uninstall doc-builder cd doc-builder git pull origin main - $PIP_OR_UV install . + uv pip install . cd .. - if [[ -n "${{ inputs.package_path }}" ]] - then - cd ${{ inputs.package_path }} - $PIP_OR_UV install .[dev] - elif [[ "${{ inputs.additional_args }}" != *"--not_python_module"* ]]; + if [[ "${{ inputs.additional_args }}" == *"--not_python_module"* ]] then - cd ${{ inputs.package }} - $PIP_OR_UV install .[dev] + echo "Not a Python module; skipping package install." + else + package_dir="${{ inputs.package_path || inputs.package }}" + # Light install when the package has a mock-deps registry entry: only its + # real (light) doc-build deps are installed and the heavy ones are mocked + # at build time. Falls back to a full [dev] install otherwise. + if doc-builder light-install ${{ env.package_name }} --check + then + uv pip install "./$package_dir" --no-deps + doc-builder light-install ${{ env.package_name }} + else + echo "Falling back to a full install." + uv pip install "./$package_dir[dev]" + fi fi - cd .. - name: Setup git run: | @@ -244,9 +214,8 @@ jobs: HF_TOKEN: ${{ secrets.hf_token }} PACKAGE_NAME: ${{ env.package_name }} run: | - pip install -U huggingface_hub cd build_dir - hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc/$PACKAGE_NAME" + uvx --from huggingface_hub hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc/$PACKAGE_NAME" cd .. - name: Push to dataset (legacy, kept for rollback safety) diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 0b7f0c16..6cddcbb0 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -46,12 +46,6 @@ on: convert_notebooks: type: boolean description: "Convert notebooks to markdown files before building docs." - # Docker image to use for the build. Set custom_container="huggingface/transformers-doc-builder" if you need - # a complete install. Default to `""` which means CI will run on the runner directly. - custom_container: - type: string - default: "" - description: "Docker image to use for the build." # Debug purposes only! doc_builder_revision: type: string @@ -66,8 +60,6 @@ on: jobs: build_pr_documentation: runs-on: ubuntu-22.04 - container: - ${{ inputs.custom_container }} defaults: run: shell: bash @@ -102,28 +94,7 @@ jobs: node-version: '20' cache-dependency-path: "kit/package-lock.json" - - name: Export ROOT_APT_GET ('apt-get' or 'sudo apt-get') - # Use `sudo` only if running on the base runner. - # When using a container, `sudo` is not needed (and not installed) - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "ROOT_APT_GET=sudo apt-get" >> $GITHUB_ENV - else - echo "ROOT_APT_GET=apt-get" >> $GITHUB_ENV - fi - - - name: Export PIP_OR_UV ('pip' or 'uv pip') - # Use `uv` only if running on the base runner. - # When using a container, `pip` has already been used to installed existing deps - # and is therefore quicker to resolve (already have some cached stuff). - run: | - if [ -z "${{ inputs.custom_container }}" ] - then - echo "PIP_OR_UV=uv pip" >> $GITHUB_ENV - else - echo "PIP_OR_UV=pip" >> $GITHUB_ENV - fi + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Set env variables run: | @@ -151,7 +122,7 @@ jobs: - name: Install libgl1 if: inputs.install_libgl1 - run: $ROOT_APT_GET install -y libgl1 + run: sudo apt-get install -y libgl1 - name: Install Rust uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master 2026-03-27 @@ -159,32 +130,33 @@ jobs: with: toolchain: stable - # Use venv in both cases - - name: Install uv - run: | - pip install -U uv - uv venv - - name: Setup environment shell: bash run: | + uv venv source .venv/bin/activate - $PIP_OR_UV uninstall doc-builder cd doc-builder git pull origin ${{ inputs.doc_builder_revision }} - $PIP_OR_UV install . + uv pip install . cd .. - if [[ -n "${{ inputs.package_path }}" ]] - then - cd ${{ inputs.package_path }} - $PIP_OR_UV install .[dev] - elif [[ "${{ inputs.additional_args }}" != *"--not_python_module"* ]]; + if [[ "${{ inputs.additional_args }}" == *"--not_python_module"* ]] then - cd ${{ inputs.package }} - $PIP_OR_UV install .[dev] + echo "Not a Python module; skipping package install." + else + package_dir="${{ inputs.package_path || inputs.package }}" + # Light install when the package has a mock-deps registry entry: only its + # real (light) doc-build deps are installed and the heavy ones are mocked + # at build time. Falls back to a full [dev] install otherwise. + if doc-builder light-install ${{ env.package_name }} --check + then + uv pip install "./$package_dir" --no-deps + doc-builder light-install ${{ env.package_name }} + else + echo "Falling back to a full install." + uv pip install "./$package_dir[dev]" + fi fi - cd .. - name: Run pre-command shell: bash diff --git a/.github/workflows/delete_old_pr_documentations.yml b/.github/workflows/delete_old_pr_documentations.yml index 35c5d494..b1e15605 100644 --- a/.github/workflows/delete_old_pr_documentations.yml +++ b/.github/workflows/delete_old_pr_documentations.yml @@ -16,8 +16,9 @@ jobs: - uses: denoland/setup-deno@11b63cf76cfcafb4e43f97b6cad24d8e8438f62d # v1.5.2 with: deno-version: v1.x + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 - name: Install hf CLI - run: pip install -U huggingface_hub + run: uv tool install huggingface_hub - run: deno run --allow-env --allow-net --allow-run --allow-read ./delete-old-prs.ts env: HF_ACCESS_TOKEN: ${{ secrets.HF_ACCESS_TOKEN }} diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml index b93ebe67..5cb8073e 100644 --- a/.github/workflows/upload_pr_documentation.yml +++ b/.github/workflows/upload_pr_documentation.yml @@ -39,14 +39,13 @@ jobs: # Uncomment the following line to use a specific revision of doc-builder # ref: fix-corrupted-zip + - uses: astral-sh/setup-uv@d31148d669074a8d0a63714ba94f3201e7020bc3 # v8.3.0 + - name: Setup environment shell: bash id: setup-env run: | - pip install black - cd doc-builder - pip install . - cd .. + uv pip install --system ./doc-builder echo "current_work_dir=$(pwd)" >> $GITHUB_OUTPUT - name: 'Download artifact' @@ -137,8 +136,7 @@ jobs: HF_TOKEN: ${{ secrets.hf_token }} run: | cd build_dir - pip install -U huggingface_hub - hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc-dev/$PACKAGE_NAME" + uvx --from huggingface_hub hf sync "./$PACKAGE_NAME" "hf://buckets/hf-doc-build/doc-dev/$PACKAGE_NAME" - name: Find doc comment uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 diff --git a/README.md b/README.md index 569e29b8..2a882bee 100644 --- a/README.md +++ b/README.md @@ -134,15 +134,22 @@ A page is reused when its generated MDX (post autodoc and internal-link resoluti doc-builder imports the documented library to read docstrings with `inspect` — necessary because libraries like `transformers` construct docstrings dynamically. Historically that meant installing the library's full dependency tree (torch, GPU wheels, custom containers) just to build documentation. -doc-builder mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed. The list of mockable dependencies per library lives in `src/doc_builder/mock_deps/.txt` and is applied automatically: +doc-builder mocks heavy dependencies instead, so only the documented library itself (and its light dependencies) needs to be installed. Each library's registry file `src/doc_builder/mock_deps/.txt` records both sides of that split: + +- bare lines name the heavy dependencies to **mock** at build time (applied automatically by `doc-builder build`/`preview`); +- `real:` lines name the light dependencies a doc build must **install for real** (things the library imports unguarded, version-pinned requirements, packages used through `isinstance` checks, ...); +- `nodeps:` lines are also installed for real but with `--no-deps`, because their own dependency trees pull in heavy packages (e.g. `accelerate`, whose install requires torch). + +`doc-builder light-install ` installs the `real:`/`nodeps:` dependencies (via `uv`), so a full light setup is: ```bash -pip install accelerate --no-deps # plus its light dependencies +uv pip install ./accelerate --no-deps +doc-builder light-install accelerate doc-builder build accelerate ~/git/accelerate/docs/source --build_dir ~/tmp/test-build # torch and deepspeed are mocked automatically (src/doc_builder/mock_deps/accelerate.txt) ``` -For a library without a registry entry (or to experiment), `--mock_deps torch,...` adds extra mocks on top of the registry. +Install the library before running `light-install`: bare registry names are pinned to the library's own declared version ranges (e.g. transformers' `tokenizers>=0.22,<=0.23`), so registry files never have to chase upstream pins. `light-install` exits with code 3 when the library has no registry entry — `light-install --check` tests this without installing anything, which is how the shared GitHub workflows decide between the light path and a full `[dev]` install. For a library without a registry entry (or to experiment), `--mock_deps torch,...` adds extra mocks on top of the registry. The registry key is the doc-builder *library name* as passed to `doc-builder build` — for namespace subpackages that is the dotted module name (e.g. `mock_deps/optimum.intel.txt`), and libraries that build under a shared name share a file (optimum-onnx builds as `optimum`). A registered name may itself be a dotted submodule (e.g. `optimum.onnxruntime`), which mocks a missing sibling inside an installed namespace package. diff --git a/src/doc_builder/commands/doc_builder_cli.py b/src/doc_builder/commands/doc_builder_cli.py index a8dea4a3..43a0959b 100644 --- a/src/doc_builder/commands/doc_builder_cli.py +++ b/src/doc_builder/commands/doc_builder_cli.py @@ -19,6 +19,7 @@ from doc_builder.commands.check_links import check_links_command_parser from doc_builder.commands.convert_doc_file import convert_command_parser from doc_builder.commands.embeddings import embeddings_command_parser +from doc_builder.commands.light_install import light_install_command_parser from doc_builder.commands.notebook_to_mdx import notebook_to_mdx_command_parser from doc_builder.commands.preview import preview_command_parser from doc_builder.commands.push import push_command_parser @@ -34,6 +35,7 @@ def main(): build_command_parser(subparsers=subparsers) check_links_command_parser(subparsers=subparsers) embeddings_command_parser(subparsers=subparsers) + light_install_command_parser(subparsers=subparsers) notebook_to_mdx_command_parser(subparsers=subparsers) style_command_parser(subparsers=subparsers) preview_command_parser(subparsers=subparsers) diff --git a/src/doc_builder/commands/light_install.py b/src/doc_builder/commands/light_install.py new file mode 100644 index 00000000..ac90daeb --- /dev/null +++ b/src/doc_builder/commands/light_install.py @@ -0,0 +1,136 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.metadata +import re +import shlex +import subprocess +import sys + +from doc_builder.mock_imports import get_registry_real_deps, registry_file_path + + +def _library_requirements(library_name): + """ + The documented library's own declared requirements (marker-free ones), as a + mapping of normalized name -> version specifier string. Empty when the library + is not installed yet. + """ + from packaging.requirements import Requirement + + requirements = {} + for dist_name in (library_name, library_name.replace(".", "-")): + try: + requires = importlib.metadata.requires(dist_name) or [] + except importlib.metadata.PackageNotFoundError: + continue + for raw in requires: + try: + requirement = Requirement(raw) + except Exception: + continue + if requirement.marker is not None: + continue # extras & platform-conditional requirements + name = re.sub(r"[-_.]+", "-", requirement.name).lower() + if requirement.specifier: + requirements[name] = str(requirement.specifier) + break + return requirements + + +def _refine(specs, requirements): + """ + Pins bare registry names to the documented library's own declared version + range, so registry files don't have to chase pins like transformers' + `tokenizers>=0.22,<=0.23`. + """ + refined = [] + for spec in specs: + if re.fullmatch(r"[A-Za-z0-9._-]+", spec): + name = re.sub(r"[-_.]+", "-", spec).lower() + if name in requirements: + refined.append(spec + requirements[name]) + continue + refined.append(spec) + return refined + + +def light_install_command(args): + """ + Installs the real (non-mocked) dependencies a light docs build of `library_name` + needs, from the `real:`/`nodeps:` lines of its mock-deps registry file. The + documented library itself is not installed — install it first with `--no-deps` + (its heavy dependencies are mocked automatically at build time); bare registry + names are then pinned to the library's own declared version ranges. + + Exits with code 3 when the library has no registry entry, so callers can fall + back to a full install: + + if doc-builder light-install my_package --check; then + uv pip install ./my_package --no-deps + doc-builder light-install my_package + else + uv pip install "./my_package[dev]" + fi + """ + if not registry_file_path(args.library_name).is_file(): + print( + f"No mock-deps registry entry for `{args.library_name}` " + f"(expected {registry_file_path(args.library_name)}); do a full install instead.", + file=sys.stderr, + ) + sys.exit(3) + if args.check: + print(f"`{args.library_name}` has a mock-deps registry entry.") + return + resolved, nodeps = get_registry_real_deps(args.library_name) + requirements = _library_requirements(args.library_name) + # target the interpreter running doc-builder, whether or not a venv is "active" + uv_pip_install = ["uv", "pip", "install", "--python", sys.executable] + for specs, extra_flags in [(resolved, []), (nodeps, ["--no-deps"])]: + if not specs: + continue + specs = _refine(specs, requirements) + if args.dry_run: + print(" ".join(["uv", "pip", "install", *extra_flags, *(shlex.quote(spec) for spec in specs)])) + continue + subprocess.run([*uv_pip_install, *extra_flags, *specs], check=True) + if not (resolved or nodeps): + print(f"`{args.library_name}` needs no real dependencies beyond the library itself.") + + +def light_install_command_parser(subparsers=None): + if subparsers is not None: + parser = subparsers.add_parser("light-install") + else: + from argparse import ArgumentParser + + parser = ArgumentParser("Doc Builder light-install command") + parser.add_argument( + "library_name", type=str, help="Library whose registered real doc-build dependencies to install." + ) + parser.add_argument( + "--check", + action="store_true", + help="Only check whether the library has a registry entry (exit code 3 when it does not).", + ) + parser.add_argument( + "--dry_run", + dest="dry_run", + action="store_true", + help="Print the install commands instead of running them.", + ) + if subparsers is not None: + parser.set_defaults(func=light_install_command) + return parser diff --git a/src/doc_builder/mock_deps/accelerate.txt b/src/doc_builder/mock_deps/accelerate.txt index abfdefde..6b30aff4 100644 --- a/src/doc_builder/mock_deps/accelerate.txt +++ b/src/doc_builder/mock_deps/accelerate.txt @@ -3,3 +3,9 @@ torch deepspeed torchvision +real:numpy +real:packaging +real:psutil +real:pyyaml +real:safetensors +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/autotrain.txt b/src/doc_builder/mock_deps/autotrain.txt index f23c8c26..25f22d7a 100644 --- a/src/doc_builder/mock_deps/autotrain.txt +++ b/src/doc_builder/mock_deps/autotrain.txt @@ -2,3 +2,18 @@ # Packages that are actually installed are never mocked. torch loguru +real:pandas +real:pyyaml +real:requests +real:huggingface_hub +nodeps:accelerate +real:safetensors +matplotlib +tensorflow +flax +jax +torchvision +torchaudio +real:transformers +real:datasets +real:scikit-learn diff --git a/src/doc_builder/mock_deps/bitsandbytes.txt b/src/doc_builder/mock_deps/bitsandbytes.txt index 3e0cd29a..6a2b25b3 100644 --- a/src/doc_builder/mock_deps/bitsandbytes.txt +++ b/src/doc_builder/mock_deps/bitsandbytes.txt @@ -1,3 +1,5 @@ # Heavy dependencies mocked when building bitsandbytes docs (see mock_imports.py). # Packages that are actually installed are never mocked. torch +real:numpy +real:packaging diff --git a/src/doc_builder/mock_deps/datasets.txt b/src/doc_builder/mock_deps/datasets.txt index cb90d499..8776ec79 100644 --- a/src/doc_builder/mock_deps/datasets.txt +++ b/src/doc_builder/mock_deps/datasets.txt @@ -1,2 +1,15 @@ # datasets docs build without mocking: its framework integrations are optional # and its required deps (pyarrow, pandas, ...) must be really installed. +real:numpy +real:pyarrow +real:dill +real:pandas +real:requests +real:tqdm +real:xxhash +real:multiprocess +real:fsspec +real:aiohttp +real:huggingface_hub +real:packaging +real:pyyaml diff --git a/src/doc_builder/mock_deps/diffusers.txt b/src/doc_builder/mock_deps/diffusers.txt index 0aabb075..24e9e678 100644 --- a/src/doc_builder/mock_deps/diffusers.txt +++ b/src/doc_builder/mock_deps/diffusers.txt @@ -1,3 +1,12 @@ # Heavy dependencies mocked when building diffusers docs (see mock_imports.py). # Packages that are actually installed are never mocked. torch +real:importlib_metadata +real:filelock +real:huggingface_hub +real:numpy +real:regex +real:requests +real:safetensors +real:pillow +real:transformers diff --git a/src/doc_builder/mock_deps/evaluate.txt b/src/doc_builder/mock_deps/evaluate.txt index ab1ceb43..8a57dc65 100644 --- a/src/doc_builder/mock_deps/evaluate.txt +++ b/src/doc_builder/mock_deps/evaluate.txt @@ -2,3 +2,14 @@ # Packages that are actually installed are never mocked. torch matplotlib +real:datasets +real:numpy +real:dill +real:pandas +real:requests +real:tqdm +real:xxhash +real:multiprocess +real:fsspec +real:huggingface_hub +real:packaging diff --git a/src/doc_builder/mock_deps/huggingface_hub.txt b/src/doc_builder/mock_deps/huggingface_hub.txt index c7c85c3e..ff513a4c 100644 --- a/src/doc_builder/mock_deps/huggingface_hub.txt +++ b/src/doc_builder/mock_deps/huggingface_hub.txt @@ -1,3 +1,10 @@ # Heavy dependencies mocked when building huggingface_hub docs (see mock_imports.py). # Packages that are actually installed are never mocked. torch +real:filelock +real:fsspec +real:packaging +real:pyyaml +real:requests +real:tqdm +real:typing_extensions diff --git a/src/doc_builder/mock_deps/kernels.txt b/src/doc_builder/mock_deps/kernels.txt new file mode 100644 index 00000000..771d86cb --- /dev/null +++ b/src/doc_builder/mock_deps/kernels.txt @@ -0,0 +1,8 @@ +# Dependencies for building kernels docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:pyyaml +real:huggingface_hub +real:packaging +real:tomli +kernels_data diff --git a/src/doc_builder/mock_deps/lerobot.txt b/src/doc_builder/mock_deps/lerobot.txt new file mode 100644 index 00000000..eb71c5d7 --- /dev/null +++ b/src/doc_builder/mock_deps/lerobot.txt @@ -0,0 +1,7 @@ +# Dependencies for building lerobot docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:numpy +real:pyyaml +real:huggingface_hub +real:packaging diff --git a/src/doc_builder/mock_deps/lighteval.txt b/src/doc_builder/mock_deps/lighteval.txt index 10ffdd12..eddd1e21 100644 --- a/src/doc_builder/mock_deps/lighteval.txt +++ b/src/doc_builder/mock_deps/lighteval.txt @@ -5,3 +5,23 @@ vllm matplotlib ray more_itertools +real:transformers +real:datasets +real:huggingface_hub +real:pydantic +real:typer +real:rich +real:colorama +real:termcolor +real:tabulate +real:aenum +real:nltk +real:pytablewriter +real:sacrebleu +real:inspect_ai +real:latex2sympy2_extended==1.0.6 +real:sympy +real:scikit-learn +real:pandas +nodeps:accelerate +real:scipy diff --git a/src/doc_builder/mock_deps/openenv.txt b/src/doc_builder/mock_deps/openenv.txt index f8d4f4e8..fe2422f4 100644 --- a/src/doc_builder/mock_deps/openenv.txt +++ b/src/doc_builder/mock_deps/openenv.txt @@ -7,3 +7,7 @@ mcp openai gradio websockets +real:requests +real:pydantic +real:aiohttp +real:typer diff --git a/src/doc_builder/mock_deps/optimum.executorch.txt b/src/doc_builder/mock_deps/optimum.executorch.txt index 1678b500..dd81e75f 100644 --- a/src/doc_builder/mock_deps/optimum.executorch.txt +++ b/src/doc_builder/mock_deps/optimum.executorch.txt @@ -1 +1,6 @@ # optimum-executorch docs build without mocks (validated with only the library installed). +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/optimum.habana.txt b/src/doc_builder/mock_deps/optimum.habana.txt index c8dbbe23..a279fbda 100644 --- a/src/doc_builder/mock_deps/optimum.habana.txt +++ b/src/doc_builder/mock_deps/optimum.habana.txt @@ -5,3 +5,10 @@ torch sklearn scipy +nodeps:optimum +real:transformers==4.55.* +nodeps:accelerate==1.10.1 +real:diffusers==0.35.1 +nodeps:sentence-transformers==3.3.1 +real:sentencepiece +real:protobuf diff --git a/src/doc_builder/mock_deps/optimum.intel.txt b/src/doc_builder/mock_deps/optimum.intel.txt index e98e836f..c77174c2 100644 --- a/src/doc_builder/mock_deps/optimum.intel.txt +++ b/src/doc_builder/mock_deps/optimum.intel.txt @@ -2,3 +2,9 @@ # Packages that are actually installed are never mocked. torch openvino +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +real:diffusers diff --git a/src/doc_builder/mock_deps/optimum.neuron.txt b/src/doc_builder/mock_deps/optimum.neuron.txt index 598800de..6aac8e08 100644 --- a/src/doc_builder/mock_deps/optimum.neuron.txt +++ b/src/doc_builder/mock_deps/optimum.neuron.txt @@ -7,3 +7,12 @@ torch_neuronx libneuronxla neuronxcc peft +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +real:diffusers==0.35.* +real:sentencepiece +nodeps:accelerate +real:datasets diff --git a/src/doc_builder/mock_deps/optimum.tpu.txt b/src/doc_builder/mock_deps/optimum.tpu.txt index d7541ffa..d6c869f0 100644 --- a/src/doc_builder/mock_deps/optimum.tpu.txt +++ b/src/doc_builder/mock_deps/optimum.tpu.txt @@ -1 +1,6 @@ # optimum-tpu docs build without mocks (validated with only the library installed). +nodeps:optimum +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/optimum.txt b/src/doc_builder/mock_deps/optimum.txt index be977b53..eba30697 100644 --- a/src/doc_builder/mock_deps/optimum.txt +++ b/src/doc_builder/mock_deps/optimum.txt @@ -5,3 +5,9 @@ torch onnxruntime onnx +nodeps:optimum-onnx +real:transformers<5 +real:numpy +real:packaging +real:huggingface_hub +nodeps:optimum diff --git a/src/doc_builder/mock_deps/peft.txt b/src/doc_builder/mock_deps/peft.txt index 84d13df1..bacebf61 100644 --- a/src/doc_builder/mock_deps/peft.txt +++ b/src/doc_builder/mock_deps/peft.txt @@ -1,3 +1,12 @@ # Heavy dependencies mocked when building peft docs (see mock_imports.py). # Packages that are actually installed are never mocked. torch +real:numpy +real:packaging +real:psutil +real:pyyaml +real:transformers +real:tqdm +real:safetensors +real:huggingface_hub +nodeps:accelerate diff --git a/src/doc_builder/mock_deps/safetensors.txt b/src/doc_builder/mock_deps/safetensors.txt index cc05ec5f..3b04f8ca 100644 --- a/src/doc_builder/mock_deps/safetensors.txt +++ b/src/doc_builder/mock_deps/safetensors.txt @@ -4,3 +4,4 @@ paddle torch tensorflow jax +real:numpy diff --git a/src/doc_builder/mock_deps/timm.txt b/src/doc_builder/mock_deps/timm.txt index 1e1b87a2..074d2ae3 100644 --- a/src/doc_builder/mock_deps/timm.txt +++ b/src/doc_builder/mock_deps/timm.txt @@ -2,3 +2,7 @@ # Packages that are actually installed are never mocked. torch torchvision +real:pyyaml +real:huggingface_hub +real:safetensors +real:numpy diff --git a/src/doc_builder/mock_deps/tokenizers.txt b/src/doc_builder/mock_deps/tokenizers.txt new file mode 100644 index 00000000..c15de9a0 --- /dev/null +++ b/src/doc_builder/mock_deps/tokenizers.txt @@ -0,0 +1,4 @@ +# Dependencies for building tokenizers docs without a full install (see mock_imports.py). +# Bare names are mocked at build time; real:/nodeps: lines are installed for real +# by `doc-builder light-install` (nodeps: = with --no-deps). +real:huggingface_hub diff --git a/src/doc_builder/mock_deps/trackio.txt b/src/doc_builder/mock_deps/trackio.txt index be6281b5..2d5da55a 100644 --- a/src/doc_builder/mock_deps/trackio.txt +++ b/src/doc_builder/mock_deps/trackio.txt @@ -2,3 +2,10 @@ # Packages that are actually installed are never mocked. torch matplotlib +real:pandas +real:pydantic +real:huggingface_hub +gradio_client +orjson +uvicorn +starlette diff --git a/src/doc_builder/mock_deps/transformers.txt b/src/doc_builder/mock_deps/transformers.txt index 19b9bf36..cc918d25 100644 --- a/src/doc_builder/mock_deps/transformers.txt +++ b/src/doc_builder/mock_deps/transformers.txt @@ -8,3 +8,17 @@ torchvision torchaudio timm deepspeed +real:filelock +real:huggingface_hub +real:numpy +real:packaging +real:pyyaml +real:regex +real:requests +real:tokenizers +real:safetensors +real:tqdm +real:sentencepiece +real:protobuf +real:pillow +real:rich diff --git a/src/doc_builder/mock_deps/trl.txt b/src/doc_builder/mock_deps/trl.txt index 5137969a..c6925132 100644 --- a/src/doc_builder/mock_deps/trl.txt +++ b/src/doc_builder/mock_deps/trl.txt @@ -1,3 +1,10 @@ # Heavy dependencies mocked when building trl docs (see mock_imports.py). # Packages that are actually installed are never mocked. torch +real:datasets +real:transformers +real:rich +nodeps:accelerate +real:jinja2 +real:psutil +nodeps:peft diff --git a/src/doc_builder/mock_imports.py b/src/doc_builder/mock_imports.py index 0b5a202b..1e6baee6 100644 --- a/src/doc_builder/mock_imports.py +++ b/src/doc_builder/mock_imports.py @@ -165,6 +165,16 @@ def __eq__(self, other): def __hash__(self): return hash(self.__display_name__) + # `issubclass(cls, torch.Tensor)` / `isinstance(x, torch.Tensor)` against a mock + # would be a TypeError (mocks are instances, not classes); scipy>=1.17 probes + # exactly this way when it sees `torch` in sys.modules. Nothing real is a + # subclass/instance of a mocked type, so answer False. + def __subclasscheck__(self, subclass): + return False + + def __instancecheck__(self, instance): + return False + def __lt__(self, other): return False @@ -329,21 +339,48 @@ def find_distributions(self, context=importlib.metadata.DistributionFinder.Conte return [] -def get_registry_mock_deps(library_name): - """ - Returns the registered list of mockable heavy dependencies for a documented - library, from `doc_builder/mock_deps/.txt` (one import name per - line, `#` comments). Empty when the library has no registry entry. - """ - registry_file = Path(__file__).parent / "mock_deps" / f"{library_name}.txt" +def registry_file_path(library_name): + """Path of a documented library's registry file (which may not exist).""" + return Path(__file__).parent / "mock_deps" / f"{library_name}.txt" + + +def _read_registry(library_name): + registry_file = registry_file_path(library_name) if not registry_file.is_file(): return [] - deps = [] + lines = [] for line in registry_file.read_text(encoding="utf-8").splitlines(): line = line.split("#", 1)[0].strip() if line: - deps.append(line) - return deps + lines.append(line) + return lines + + +def get_registry_mock_deps(library_name): + """ + Returns the registered list of mockable heavy dependencies for a documented + library, from `doc_builder/mock_deps/.txt` (one import name per + line, `#` comments). Lines prefixed with `real:` or `nodeps:` list packages to + install for real instead (see `get_registry_real_deps`) and are skipped here. + Empty when the library has no registry entry. + """ + return [line for line in _read_registry(library_name) if not line.startswith(("real:", "nodeps:"))] + + +def get_registry_real_deps(library_name): + """ + Returns the registered dependencies a light docs build must install for real + (not mock), as a tuple `(resolved, nodeps)` of pip requirement lists: + `real:` lines resolve normally, `nodeps:` lines must be installed + with `--no-deps` (their own dependency trees pull in heavy packages). + """ + resolved, nodeps = [], [] + for line in _read_registry(library_name): + if line.startswith("real:"): + resolved.append(line[len("real:") :].strip()) + elif line.startswith("nodeps:"): + nodeps.append(line[len("nodeps:") :].strip()) + return resolved, nodeps def mock_deps(packages): diff --git a/tests/test_mock_imports.py b/tests/test_mock_imports.py index bd075e3d..6ceb9d04 100644 --- a/tests/test_mock_imports.py +++ b/tests/test_mock_imports.py @@ -105,6 +105,13 @@ class Model(module.nn.Module): # class-level fallback: the real base (e.g. nn.Module) would provide `forward` self.assertIsNotNone(getattr(Model, "forward", None)) + def test_isinstance_issubclass_against_mock(self): + module = importlib.import_module(PKG) + # scipy>=1.17 does `issubclass(cls, torch.Tensor)` when torch is in + # sys.modules; a mock must answer False, not raise TypeError + self.assertFalse(issubclass(int, module.Tensor)) + self.assertFalse(isinstance(3, module.Tensor)) + def test_dotted_submodule_mock(self): # a registered dotted name mocks a missing submodule of a real (namespace) # package, like `optimum.onnxruntime` when only `optimum` is installed @@ -136,6 +143,18 @@ def test_registry_mock_deps(self): # validated as needing no mocks -> empty but present self.assertEqual(get_registry_mock_deps("datasets"), []) + def test_registry_real_deps(self): + from doc_builder.mock_imports import get_registry_mock_deps, get_registry_real_deps + + resolved, nodeps = get_registry_real_deps("peft") + self.assertIn("transformers", resolved) + self.assertIn("accelerate", nodeps) # accelerate's own deps include torch + # real:/nodeps: lines are install directives, not mocks + mocks = get_registry_mock_deps("peft") + self.assertIn("torch", mocks) + self.assertFalse(any(m.startswith(("real:", "nodeps:")) for m in mocks)) + self.assertEqual(get_registry_real_deps("not-a-real-library"), ([], [])) + def test_installed_packages_are_not_mocked(self): # `json` is really installed, so it must be skipped mocked = mock_deps(["json"])