From 3300f52aea8dc8c083d66e23db543b6b0b991595 Mon Sep 17 00:00:00 2001 From: Mishig Date: Sun, 5 Jul 2026 17:54:39 +0200 Subject: [PATCH] feat: page-level HTML build cache backed by HF storage buckets Building the HTML docs of a large library (transformers: ~400 en pages, 14 + 18 min per main build) prerenders every page even when a commit changed one of them. This adds an opt-in page-level cache (--html_page_cache ) so the SvelteKit build only prerenders pages whose content actually changed: - key = sha256(generated mdx, kit tree hash, package, language, doc-builder version). Keying on the post-autodoc, post-link-resolution mdx means docstring changes invalidate exactly the affected pages. - keys are version-normalized: resolved internal links embed /docs/{pkg}/{version}/{lang}/, which would make otherwise-identical pages differ between main and pr_N builds. On a cross-version reuse, page URLs are rewritten to the target version while asset URLs stay pinned to the source version (its content-hashed assets remain hosted by the additive doc bucket; chunk-to-chunk imports resolve relative to the chunk's own URL). - storage: manifest per (package, language) + content-addressed blobs, over a local dir or an HF storage bucket via huggingface_hub. - cache failures are never fatal (degrade to building the pages). - shared workflows: main builds read+write the hf-doc-build/ doc-build-cache bucket; PR builds are read-only (no secrets, public bucket), so untrusted code cannot poison the cache. Verified on the accelerate e2e (local dir and a real bucket): cold populates 48 pages with output byte-identical to an uncached build; warm reuses 46/48 (the 2 misses are the docstring pages containing nondeterministic python object addresses - correct invalidation); single-file change rebuilds exactly that page, mixed output equivalent to a full fresh build; cross-version (--version pr_5) reuses 35/48 with page links rewritten to pr_5 and all asset refs pinned to main. Co-Authored-By: Claude Fable 5 --- .../workflows/build_main_documentation.yml | 4 +- .github/workflows/build_pr_documentation.yml | 2 +- README.md | 18 ++ src/doc_builder/build_cache.py | 297 ++++++++++++++++++ src/doc_builder/commands/build.py | 64 +++- tests/test_build_cache.py | 108 +++++++ 6 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 src/doc_builder/build_cache.py create mode 100644 tests/test_build_cache.py diff --git a/.github/workflows/build_main_documentation.yml b/.github/workflows/build_main_documentation.yml index 0d82f249..e698a8d5 100644 --- a/.github/workflows/build_main_documentation.yml +++ b/.github/workflows/build_main_documentation.yml @@ -211,11 +211,13 @@ jobs: shell: bash env: NODE_OPTIONS: --max-old-space-size=6656 + # needed for --html_page_cache_write (trusted main builds only) + HF_TOKEN: ${{ secrets.hf_token }} run: | source .venv/bin/activate echo "doc_folder has been set to ${{ env.doc_folder }}" cd doc-builder - args="--build_dir ../build_dir --clean --html ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}" + args="--build_dir ../build_dir --clean --html --html_page_cache hf://buckets/hf-doc-build/doc-build-cache --html_page_cache_write ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}" if [ ! -z "${{ inputs.notebook_folder }}" ]; then diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index 57902a35..0b7f0c16 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -220,7 +220,7 @@ jobs: log_file="../build_dir/doc-build.log" : > "$log_file" set -o pipefail - args="--build_dir ../build_dir --clean --version pr_${{ inputs.pr_number }} --html ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}" + args="--build_dir ../build_dir --clean --version pr_${{ inputs.pr_number }} --html --html_page_cache hf://buckets/hf-doc-build/doc-build-cache ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}" if [ -z "${{ inputs.languages }}" ]; then diff --git a/README.md b/README.md index 9a680826..b6ffc0ec 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,24 @@ doc-builder build hub ~/git/hub-docs/docs/source --build_dir ~/tmp/test-build -- **Important**: When specifying a semantic version with `--version`, it **must start with the letter "v"** (e.g., `v1.0.0`, `v2.3.1`). For branch names like `main`, `master`, or other default branches, the "v" prefix is not required. +### Page-level HTML build cache + +Building the HTML docs of a large library (e.g. `transformers`) prerenders hundreds of pages, even when a commit only changed one of them. `--html_page_cache` enables a page-level cache so that only pages whose generated content changed are prerendered again: + +```bash +# cache in a Hugging Face storage bucket (https://huggingface.co/docs/hub/storage-buckets) ... +doc-builder build datasets ~/git/datasets/docs/source --build_dir ~/tmp/test-build --html \ + --html_page_cache hf://buckets/hf-doc-build/doc-build-cache + +# ... or in a local directory +doc-builder build datasets ~/git/datasets/docs/source --build_dir ~/tmp/test-build --html \ + --html_page_cache ~/tmp/doc-page-cache --html_page_cache_write +``` + +A page is reused when its generated MDX (post autodoc and internal-link resolution — so docstring changes invalidate exactly the affected pages), the `kit` folder, and the doc-builder version are unchanged. Keys are version-normalized: pages that only differ by the built version (e.g. `main` vs `pr_123` in resolved internal links) share a cache entry, and on a cross-version reuse the page URLs are rewritten to the target version while asset URLs stay pinned to the source version, whose content-hashed assets remain hosted. + +`--html_page_cache_write` also stores freshly built pages. Only enable it on trusted builds (the shared GitHub workflows enable it on main-branch builds and keep PR builds read-only), so that untrusted code cannot poison the cache. Cache failures are never fatal: any error degrades to building the affected pages. + ### Notebook conversion `doc-builder` can also automatically convert some of the documentation guides or tutorials into notebooks. This requires two steps: diff --git a/src/doc_builder/build_cache.py b/src/doc_builder/build_cache.py new file mode 100644 index 00000000..b9d480b5 --- /dev/null +++ b/src/doc_builder/build_cache.py @@ -0,0 +1,297 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Page-level HTML build cache for `doc-builder build --html`. + +Building the HTML docs of a large library (e.g. transformers) prerenders hundreds of +pages even when a commit only changed one of them. This module caches each built page +keyed by the content that produced it, so subsequent builds only run the SvelteKit +build for pages whose content actually changed. + +Design: + +- A page's cache key is `sha256` of its **generated mdx content** (i.e. after autodoc + and internal-link resolution, so docstring changes from library code invalidate + exactly the affected pages), the **kit tree hash** (any component/dependency change + invalidates everything), the doc-builder version and the language. +- The mdx content is *version-normalized* before hashing: resolved internal links and + base paths embed `/docs/{package}/{version}/{language}/`, which would make otherwise + identical pages differ between e.g. `main` and `pr_123` builds. The version segment + is replaced with a placeholder so those pages share a key. +- The cache stores a manifest per `(package, language)` mapping each page to + `{key, blob, source_version}`, plus content-addressed HTML blobs. +- On a same-version hit, the cached HTML is reused byte-identical. On a cross-version + hit, page-URL occurrences of the source base path are rewritten to the current + version, while asset URLs (`.../_app/...`) stay pinned to the source version: the + serving bucket is additive, so the source version's content-hashed assets remain + hosted, and chunk-to-chunk imports resolve relative to the chunk's own URL. +- Storage backends: a local directory (used in tests / local builds) or a Hugging Face + storage bucket (`hf://buckets/...`, see https://huggingface.co/docs/hub/storage-buckets). +- Cache failures are never fatal: any error degrades to building the affected pages. + +Security note: cache writes should only be enabled on trusted builds (e.g. main-branch +builds). PR builds should be read-only so that a malicious PR cannot poison pages that +a later trusted build would reuse. +""" + +import hashlib +import json +import re +import shutil +import tempfile +import traceback +from dataclasses import dataclass +from pathlib import Path + +from . import __version__ + +VERSION_PLACEHOLDER = "__DOC_BUILDER_VERSION__" + +# node_modules etc. are not part of the staged kit (see commands/build.py) and local +# build artifacts must not influence the cache key +KIT_HASH_IGNORE_DIRS = {"node_modules", ".svelte-kit", "build"} + + +def hash_kit_tree(kit_folder): + """ + Deterministic hash of the kit folder contents (everything that is staged for the + SvelteKit build: sources, preprocessors, configs, lockfile). + """ + kit_folder = Path(kit_folder) + sha = hashlib.sha256() + files = [ + f + for f in sorted(kit_folder.rglob("*")) + if f.is_file() and not any(part in KIT_HASH_IGNORE_DIRS for part in f.relative_to(kit_folder).parts) + ] + for f in files: + sha.update(f.relative_to(kit_folder).as_posix().encode("utf-8")) + sha.update(b"\0") + sha.update(f.read_bytes()) + sha.update(b"\0") + return sha.hexdigest() + + +def normalize_version(text, package, version, language): + """ + Replaces the versioned doc base path (embedded by internal-link resolution and the + SvelteKit base path) with a placeholder, so content that only differs by the built + version compares equal. + """ + return text.replace(f"/docs/{package}/{version}/{language}/", f"/docs/{package}/{VERSION_PLACEHOLDER}/{language}/") + + +def page_cache_key(mdx_content, kit_hash, package, version, language): + """ + Cache key for one page. `mdx_content` is the final generated mdx (post autodoc and + link resolution). + """ + normalized = normalize_version(mdx_content, package, version, language) + sha = hashlib.sha256() + for part in (normalized, kit_hash, package, language, __version__): + sha.update(part.encode("utf-8")) + sha.update(b"\0") + return sha.hexdigest() + + +def rewrite_cached_html(html, package, source_version, target_version, language): + """ + Rewrites a cached page built for `source_version` so it can be served under + `target_version`: page URLs (resolved internal links, SvelteKit base config, + metadata) move to the target version, while asset URLs (`.../_app/...`) keep + pointing at the source version, whose content-hashed assets remain hosted. + """ + if source_version == target_version: + return html + source_base = f"/docs/{package}/{source_version}/{language}/" + target_base = f"/docs/{package}/{target_version}/{language}/" + # negative lookahead: leave asset URLs on the source version + return re.sub(re.escape(source_base) + r"(?!_app/)", target_base.replace("\\", r"\\\\"), html) + + +@dataclass +class FetchedPage: + page: str # rel mdx posix path, e.g. "quicktour.mdx" + html_path: Path # local path of the fetched (and possibly rewritten) html + source_version: str + + +class PageCache: + """ + Page-level HTML cache over a local directory or an HF storage bucket. + + Layout under the cache root: + //manifest.json + blobs//.html + """ + + def __init__(self, cache_url, package, version, language, write=False): + self.package = package + self.version = version + self.language = language + self.write = write + self.is_bucket = cache_url.startswith("hf://") + self.cache_url = cache_url.rstrip("/") + if not self.is_bucket: + Path(self.cache_url).mkdir(parents=True, exist_ok=True) + self._workdir = Path(tempfile.mkdtemp(prefix="doc_builder_page_cache_")) + + # ------------------------------------------------------------------ paths + @property + def _manifest_rel(self): + return f"{self.package}/{self.language}/manifest.json" + + def _blob_rel(self, key): + return f"blobs/{key[:2]}/{key}.html" + + # ---------------------------------------------------------------- backend + def _bucket_id_and_prefix(self): + # hf://buckets//[/] + parts = self.cache_url.removeprefix("hf://buckets/").split("/") + bucket_id = "/".join(parts[:2]) + prefix = "/".join(parts[2:]) + return bucket_id, prefix + + def _remote_path(self, rel): + _, prefix = self._bucket_id_and_prefix() + return f"{prefix}/{rel}" if prefix else rel + + def _download(self, rel_paths, dest_paths): + """Downloads `rel_paths` to `dest_paths`. Returns the successfully fetched indices.""" + if not rel_paths: + return [] + if not self.is_bucket: + fetched = [] + for i, (rel, dest) in enumerate(zip(rel_paths, dest_paths, strict=True)): + src = Path(self.cache_url) / rel + if src.is_file(): + Path(dest).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dest) + fetched.append(i) + return fetched + from huggingface_hub import download_bucket_files + + bucket_id, _ = self._bucket_id_and_prefix() + for dest in dest_paths: + Path(dest).parent.mkdir(parents=True, exist_ok=True) + try: + download_bucket_files( + bucket_id, + files=[(self._remote_path(rel), str(dest)) for rel, dest in zip(rel_paths, dest_paths, strict=True)], + ) + except Exception: + # transient error: fall back to per-file downloads + for rel, dest in zip(rel_paths, dest_paths, strict=True): + try: + download_bucket_files(bucket_id, files=[(self._remote_path(rel), str(dest))]) + except Exception: + pass + # `download_bucket_files` skips files missing on the remote instead of raising, + # so report success based on what actually landed on disk + return [i for i, dest in enumerate(dest_paths) if Path(dest).is_file()] + + def _upload(self, items): + """Uploads `(local_path, rel_path)` pairs.""" + if not items: + return + if not self.is_bucket: + for local, rel in items: + dest = Path(self.cache_url) / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(local, dest) + return + from huggingface_hub import batch_bucket_files + + bucket_id, _ = self._bucket_id_and_prefix() + batch_bucket_files(bucket_id, add=[(str(local), self._remote_path(rel)) for local, rel in items]) + + # ----------------------------------------------------------------- public + def load_manifest(self): + """ + Returns the manifest for this package/language: `{page: {key, blob, source_version}}`. + Empty on any failure (treated as a cold cache). + """ + try: + dest = self._workdir / "manifest.json" + fetched = self._download([self._manifest_rel], [dest]) + if not fetched: + return {} + return json.loads(dest.read_text(encoding="utf-8")) + except Exception: + print("[page-cache] could not load the cache manifest, treating the cache as cold") + traceback.print_exc() + return {} + + def fetch_pages(self, wanted, manifest): + """ + `wanted` is `{page: key}` for the current build. Fetches every page whose key + matches the manifest and returns `{page: FetchedPage}` for the successful ones. + Cross-version entries are rewritten for the current version. + """ + hits = { + page: manifest[page] + for page, key in wanted.items() + if page in manifest and manifest[page].get("key") == key + } + if not hits: + return {} + try: + pages = list(hits.keys()) + rel_paths = [self._blob_rel(hits[page]["blob"]) for page in pages] + dest_paths = [self._workdir / "fetched" / page for page in pages] + fetched_indices = self._download(rel_paths, dest_paths) + fetched = {} + for i in fetched_indices: + page = pages[i] + entry = hits[page] + source_version = entry.get("source_version", self.version) + if source_version != self.version: + html = dest_paths[i].read_text(encoding="utf-8") + html = rewrite_cached_html(html, self.package, source_version, self.version, self.language) + dest_paths[i].write_text(html, encoding="utf-8") + fetched[page] = FetchedPage(page=page, html_path=dest_paths[i], source_version=source_version) + return fetched + except Exception: + print("[page-cache] fetching cached pages failed, building all pages") + traceback.print_exc() + return {} + + def store_pages(self, built, manifest, reused): + """ + Uploads freshly built pages and the updated manifest (only when `write=True`). + + - `built` is `{page: (key, html_path)}` for pages built in this run. + - `manifest` is the previously loaded manifest. + - `reused` is `{page: FetchedPage}` for cache hits merged into this build. + """ + if not self.write: + return + try: + uploads = [] + new_manifest = {} + for page in reused: + new_manifest[page] = dict(manifest[page]) + for page, (key, html_path) in built.items(): + blob = key + uploads.append((html_path, self._blob_rel(blob))) + new_manifest[page] = {"key": key, "blob": blob, "source_version": self.version} + manifest_path = self._workdir / "manifest_out.json" + manifest_path.write_text(json.dumps(new_manifest, indent=1, sort_keys=True), encoding="utf-8") + uploads.append((manifest_path, self._manifest_rel)) + self._upload(uploads) + print(f"[page-cache] stored {len(built)} built page(s) and the updated manifest") + except Exception: + print("[page-cache] storing built pages failed (the build itself is unaffected)") + traceback.print_exc() diff --git a/src/doc_builder/commands/build.py b/src/doc_builder/commands/build.py index 6067ed16..eb42581a 100644 --- a/src/doc_builder/commands/build.py +++ b/src/doc_builder/commands/build.py @@ -22,6 +22,7 @@ from pathlib import Path from doc_builder import build_doc, update_versions_file +from doc_builder.build_cache import PageCache, hash_kit_tree, page_cache_key from doc_builder.utils import ( get_default_branch_name, get_doc_config, @@ -213,8 +214,38 @@ def build_command(args): # If asked, convert the MDX files into HTML files. if args.html: + package_name = os.environ.get("package_name") or args.library_name + + # Page-level HTML cache: reuse previously built pages whose generated mdx (and + # the kit itself) did not change, so the SvelteKit build only renders the rest. + page_cache = None + page_keys, manifest, reused = {}, {}, {} + if args.html_page_cache: + page_cache = PageCache( + args.html_page_cache, + package=package_name, + version=version, + language=args.language, + write=args.html_page_cache_write, + ) + kit_hash = hash_kit_tree(kit_folder) + for mdx_file in output_path.rglob("*.mdx"): + rel = mdx_file.relative_to(output_path).as_posix() + page_keys[rel] = page_cache_key( + mdx_file.read_text(encoding="utf-8"), kit_hash, package_name, version, args.language + ) + manifest = page_cache.load_manifest() + reused = page_cache.fetch_pages(page_keys, manifest) + print(f"[page-cache] reusing {len(reused)}/{len(page_keys)} cached page(s)") + with tempfile.TemporaryDirectory() as tmp_dir: - kit_dir, _, markdown_exports = stage_kit_routes(kit_folder, tmp_dir, output_path) + kit_dir, routes_dir, markdown_exports = stage_kit_routes(kit_folder, tmp_dir, output_path) + + # Cached pages don't need to be prerendered again + for page in reused: + staged_route = Path(sveltify_file_route(routes_dir / page)) + if staged_route.is_file(): + staged_route.unlink() # Move the objects.inv file at the root if not args.not_python_module: @@ -230,6 +261,23 @@ def build_command(args): # Copy result back in the build_dir. shutil.rmtree(output_path) shutil.copytree(kit_dir / "build", output_path) + + # Merge the cached pages into the build output + for page, fetched in reused.items(): + cached_dest = output_path / (page.removesuffix(".mdx") + ".html") + cached_dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(fetched.html_path, cached_dest) + + # Store the freshly built pages (no-op unless --html_page_cache_write) + if page_cache is not None: + built = {} + for page, key in page_keys.items(): + if page in reused: + continue + built_html = output_path / (page.removesuffix(".mdx") + ".html") + if built_html.is_file(): + built[page] = (key, built_html) + page_cache.store_pages(built, manifest, reused) # write markdown routes alongside the generated html output for relative_path, content in markdown_exports: markdown_dest = output_path / relative_path @@ -300,6 +348,20 @@ def build_command_parser(subparsers=None): action="store_true", help="Emit conversion warnings, such as bare asserts in runnable markdown code blocks.", ) + parser.add_argument( + "--html_page_cache", + type=str, + default=None, + help="Page-level HTML build cache location: an HF storage bucket path (e.g. " + "`hf://buckets/hf-doc-build/doc-build-cache`) or a local directory. Only used with --html. Pages whose " + "generated mdx did not change are reused from the cache instead of being prerendered again.", + ) + parser.add_argument( + "--html_page_cache_write", + action="store_true", + help="Also write freshly built pages to the page cache. Enable only on trusted builds (e.g. main-branch " + "builds), never on PR builds, so that untrusted code cannot poison the cache.", + ) if subparsers is not None: parser.set_defaults(func=build_command) return parser diff --git a/tests/test_build_cache.py b/tests/test_build_cache.py new file mode 100644 index 00000000..4261117d --- /dev/null +++ b/tests/test_build_cache.py @@ -0,0 +1,108 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest +from pathlib import Path + +from doc_builder.build_cache import PageCache, hash_kit_tree, page_cache_key, rewrite_cached_html + + +class BuildCacheTester(unittest.TestCase): + def test_page_cache_key_is_stable(self): + key1 = page_cache_key("# Hello", "kithash", "accelerate", "main", "en") + key2 = page_cache_key("# Hello", "kithash", "accelerate", "main", "en") + self.assertEqual(key1, key2) + self.assertNotEqual(key1, page_cache_key("# Hello!", "kithash", "accelerate", "main", "en")) + self.assertNotEqual(key1, page_cache_key("# Hello", "otherkit", "accelerate", "main", "en")) + self.assertNotEqual(key1, page_cache_key("# Hello", "kithash", "accelerate", "main", "fr")) + + def test_page_cache_key_is_version_normalized(self): + # resolved internal links embed the built version; identical content built for + # different versions must share a key + main_mdx = "See [Trainer](/docs/transformers/main/en/main_classes/trainer#transformers.Trainer)" + pr_mdx = "See [Trainer](/docs/transformers/pr_123/en/main_classes/trainer#transformers.Trainer)" + key_main = page_cache_key(main_mdx, "kithash", "transformers", "main", "en") + key_pr = page_cache_key(pr_mdx, "kithash", "transformers", "pr_123", "en") + self.assertEqual(key_main, key_pr) + + def test_rewrite_cached_html(self): + html = ( + '' + 'quicktour' + '' + ) + rewritten = rewrite_cached_html(html, "transformers", "main", "pr_123", "en") + # asset URLs stay pinned to the source version (still hosted there) + self.assertIn("/docs/transformers/main/en/_app/immutable/entry/start.abc.js", rewritten) + # page URLs move to the target version + self.assertIn('href="/docs/transformers/pr_123/en/quicktour"', rewritten) + self.assertIn('base: "/docs/transformers/pr_123/en/page"', rewritten) + self.assertNotIn('href="/docs/transformers/main/en/quicktour"', rewritten) + # same-version rewrite is a no-op + self.assertEqual(rewrite_cached_html(html, "transformers", "main", "main", "en"), html) + + def test_hash_kit_tree_ignores_artifacts(self): + with tempfile.TemporaryDirectory() as tmp: + kit = Path(tmp) + (kit / "src").mkdir() + (kit / "src" / "app.css").write_text("body {}") + base_hash = hash_kit_tree(kit) + (kit / "node_modules").mkdir() + (kit / "node_modules" / "dep.js").write_text("x") + (kit / ".svelte-kit").mkdir() + (kit / ".svelte-kit" / "gen.js").write_text("y") + self.assertEqual(hash_kit_tree(kit), base_hash) + (kit / "src" / "app.css").write_text("body { color: red }") + self.assertNotEqual(hash_kit_tree(kit), base_hash) + + def test_local_cache_roundtrip(self): + with tempfile.TemporaryDirectory() as cache_dir, tempfile.TemporaryDirectory() as build_dir: + html = Path(build_dir) / "quicktour.html" + html.write_text('home', encoding="utf-8") + key = page_cache_key("mdx content", "kithash", "accelerate", "main", "en") + + writer = PageCache(cache_dir, package="accelerate", version="main", language="en", write=True) + self.assertEqual(writer.load_manifest(), {}) + writer.store_pages({"quicktour.mdx": (key, html)}, manifest={}, reused={}) + + reader = PageCache(cache_dir, package="accelerate", version="main", language="en") + manifest = reader.load_manifest() + self.assertIn("quicktour.mdx", manifest) + + # matching key -> hit, byte-identical + fetched = reader.fetch_pages({"quicktour.mdx": key}, manifest) + self.assertEqual(set(fetched), {"quicktour.mdx"}) + self.assertEqual(fetched["quicktour.mdx"].html_path.read_text(encoding="utf-8"), html.read_text()) + + # changed content -> miss + other_key = page_cache_key("changed mdx", "kithash", "accelerate", "main", "en") + self.assertEqual(reader.fetch_pages({"quicktour.mdx": other_key}, manifest), {}) + + # cross-version hit -> page urls rewritten + pr_reader = PageCache(cache_dir, package="accelerate", version="pr_5", language="en") + fetched = pr_reader.fetch_pages({"quicktour.mdx": key}, manifest) + self.assertEqual(set(fetched), {"quicktour.mdx"}) + self.assertIn( + "/docs/accelerate/pr_5/en/index", fetched["quicktour.mdx"].html_path.read_text(encoding="utf-8") + ) + + def test_write_disabled_does_not_store(self): + with tempfile.TemporaryDirectory() as cache_dir, tempfile.TemporaryDirectory() as build_dir: + html = Path(build_dir) / "index.html" + html.write_text("

hi

", encoding="utf-8") + key = page_cache_key("mdx", "kithash", "accelerate", "main", "en") + cache = PageCache(cache_dir, package="accelerate", version="main", language="en", write=False) + cache.store_pages({"index.mdx": (key, html)}, manifest={}, reused={}) + self.assertEqual(cache.load_manifest(), {})