align snapshot_download to respect hfh latest version (#14118)

* align snapshot_download to respect hfh latest version

* fix

* compute the same snapshot_download patterns offline as online

Instead of returning the cached snapshot folder without validation when
offline, compute the same allow/ignore patterns as the online path
(sourcing the file listing from the cached snapshot instead of
model_info) and let snapshot_download validate the cached snapshot
against them, so an interrupted download surfaces hfh's
IncompleteSnapshotError instead of failing later at model load time.
Also revert the _get_checkpoint_shard_files divergence: its patterns
were already identical in both modes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* match missing-shard test on the shard filename

huggingface_hub>=1.22.0 reports a missing cached shard via
IncompleteSnapshotError (repo-relative path) before diffusers' own
cached-folder check (absolute path) runs, so assert on the shard's
filename, which both messages contain.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* use get_cached_repo_tree

* remove the requirement decorator from tests

* force_download=True

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: sayakpaul

setup.py

@@ -103,7 +103,7 @@
     "ftfy",
     "hf-doc-builder>=0.3.0",
     "httpx<1.0.0",
-    "huggingface-hub>=0.34.0,<2.0",
+    "huggingface-hub>=1.23.0,<2.0",
     "requests-mock==1.10.0",
     "importlib_metadata",
     "invisible-watermark>=0.2.0",

src/diffusers/dependency_versions_table.py

@@ -10,7 +10,7 @@
     "ftfy": "ftfy",
     "hf-doc-builder": "hf-doc-builder>=0.3.0",
     "httpx": "httpx<1.0.0",
-    "huggingface-hub": "huggingface-hub>=0.34.0,<2.0",
+    "huggingface-hub": "huggingface-hub>=1.23.0,<2.0",
     "requests-mock": "requests-mock==1.10.0",
     "importlib_metadata": "importlib_metadata",
     "invisible-watermark": "invisible-watermark>=0.2.0",

src/diffusers/pipelines/pipeline_utils.py

@@ -33,12 +33,19 @@
     DDUFEntry,
     ModelCard,
     create_repo,
+    get_cached_repo_tree,
     hf_hub_download,
     model_info,
     read_dduf_file,
     snapshot_download,
 )
-from huggingface_hub.utils import HfHubHTTPError, OfflineModeIsEnabled, validate_hf_hub_args
+from huggingface_hub.errors import CachedRepoTreeNotFoundError
+from huggingface_hub.utils import (
+    HfHubHTTPError,
+    LocalEntryNotFoundError,
+    OfflineModeIsEnabled,
+    validate_hf_hub_args,
+)
 from packaging import version
 from tqdm.auto import tqdm
 from typing_extensions import Self
@@ -1658,10 +1665,35 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
                 force_download=force_download,
                 token=token,
             )
+            filenames = {sibling.rfilename for sibling in info.siblings}
+        else:
+            # We are offline (either the user asked for `local_files_only` or the `model_info` call above
+            # failed), so the repo's file listing needed to compute the allow/ignore patterns below cannot be
+            # fetched from the Hub. Read it from the local cache with `get_cached_repo_tree` instead: it returns
+            # the file listing that a previous online call cached, so both code paths compute the same patterns.
+            # If nothing is cached for this repo, skip straight to `snapshot_download` below, which raises the
+            # appropriate offline not-found error.
+            try:
+                config_file = hf_hub_download(
+                    pretrained_model_name,
+                    cls.config_name,
+                    cache_dir=cache_dir,
+                    revision=revision,
+                    token=token,
+                    local_files_only=True,
+                )
+                filenames = {
+                    f.path for f in get_cached_repo_tree(pretrained_model_name, revision=revision, cache_dir=cache_dir)
+                }
+            except (LocalEntryNotFoundError, CachedRepoTreeNotFoundError):
+                config_file = None
+
+        if config_file is not None:
+            snapshot_folder = Path(config_file).parent
+
             config_dict = cls._dict_from_json_file(config_file)
             ignore_filenames = config_dict.pop("_ignore_files", [])
 
-            filenames = {sibling.rfilename for sibling in info.siblings}
             if variant is not None and _check_legacy_sharding_variant_format(filenames=filenames, variant=variant):
                 warn_msg = (
                     f"Warning: The repository contains sharded checkpoints for variant '{variant}' maybe in a deprecated format. "
@@ -1676,9 +1708,11 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
                 logger.warning(warn_msg)
 
             filenames = set(filenames) - set(ignore_filenames)
-            if revision in DEPRECATED_REVISION_ARGS and version.parse(
-                version.parse(__version__).base_version
-            ) >= version.parse("0.22.0"):
+            if (
+                not local_files_only
+                and revision in DEPRECATED_REVISION_ARGS
+                and version.parse(version.parse(__version__).base_version) >= version.parse("0.22.0")
+            ):
                 warn_deprecated_model_variant(pretrained_model_name, token, variant, revision, filenames)
 
             custom_components, folder_names = _get_custom_components_and_folders(
@@ -1769,7 +1803,6 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
             expected_files = [f for f in filenames if not any(p.match(f) for p in re_ignore_pattern)]
             expected_files = [f for f in expected_files if any(p.match(f) for p in re_allow_pattern)]
 
-            snapshot_folder = Path(config_file).parent
             pipeline_is_cached = all((snapshot_folder / f).is_file() for f in expected_files)
 
             if pipeline_is_cached and not force_download:
@@ -1818,7 +1851,7 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
 
             return cached_folder
 
-        except FileNotFoundError:
+        except FileNotFoundError as e:
             # Means we tried to load pipeline with `local_files_only=True` but the files have not been found in local cache.
             # This can happen in two cases:
             # 1. If the user passed `local_files_only=True`                    => we raise the error directly
@@ -1829,9 +1862,9 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
             else:
                 # 2. we forced `local_files_only=True` when `model_info` failed
                 raise EnvironmentError(
-                    f"Cannot load model {pretrained_model_name}: model is not cached locally and an error occurred"
-                    " while trying to fetch metadata from the Hub. Please check out the root cause in the stacktrace"
-                    " above."
+                    f"Cannot load model {pretrained_model_name}: the model is not fully cached locally ({e}) and an"
+                    " error occurred while trying to fetch metadata from the Hub. Please check out the root cause in"
+                    " the stacktrace above."
                 ) from model_info_call_error
 
     @classmethod

tests/models/test_modeling_common.py

@@ -161,9 +161,11 @@ def test_local_files_only_with_sharded_checkpoint(self):
                     repo_id, subfolder="transformer", cache_dir=tmpdir, local_files_only=True
                 )
 
-            # Verify error mentions the missing shard
+            # Verify error mentions the missing shard. `huggingface_hub>=1.22.0` reports it via
+            # `IncompleteSnapshotError` (repo-relative path), older versions via diffusers' own check on the
+            # cached folder (absolute path), so match on the shard's filename.
             error_msg = str(context.value)
-            assert cached_shard_file in error_msg or "required according to the checkpoint index" in error_msg, (
+            assert os.path.basename(cached_shard_file) in error_msg, (
                 f"Expected error about missing shard, got: {error_msg}"
             )
 

tests/pipelines/test_pipelines.py

@@ -25,6 +25,7 @@
 import unittest
 import unittest.mock as mock
 import warnings
+from pathlib import Path
 
 import numpy as np
 import PIL.Image
@@ -504,6 +505,53 @@ def test_local_files_only_are_used_when_no_internet(self):
                 if p1.data.ne(p2.data).sum() > 0:
                     assert False, "Parameters not the same!"
 
+    def test_local_files_only_uses_same_snapshot_download_patterns(self):
+        # diffusers downloads a filtered subset of a repo (skipping e.g. `.gitattributes`). Newer
+        # `huggingface_hub` versions validate that a cached snapshot contains every file matching the
+        # requested patterns under `local_files_only=True`, so an offline
+        # `snapshot_download(allow_patterns=None)` would wrongly expect the whole repo. The offline path
+        # must compute the same patterns as the online one (from the cached snapshot instead of
+        # `model_info`). See https://github.com/huggingface/diffusers/issues/14117
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with mock.patch(
+                "diffusers.pipelines.pipeline_utils.snapshot_download", side_effect=snapshot_download
+            ) as mock_snapshot_download:
+                cached_folder = DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch", cache_dir=tmpdirname
+                )
+                online_kwargs = mock_snapshot_download.call_args.kwargs
+
+                # `force_download=True` skips the fully-cached early return so `snapshot_download` runs
+                # offline and validates the computed patterns against the cached snapshot.
+                offline_folder = DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch",
+                    cache_dir=tmpdirname,
+                    local_files_only=True,
+                    force_download=True,
+                )
+                offline_kwargs = mock_snapshot_download.call_args.kwargs
+
+            assert os.path.samefile(offline_folder, cached_folder)
+            assert set(offline_kwargs["allow_patterns"]) == set(online_kwargs["allow_patterns"])
+            assert set(offline_kwargs["ignore_patterns"]) == set(online_kwargs["ignore_patterns"])
+
+    def test_local_files_only_raises_for_snapshot_with_missing_weights(self):
+        # An interrupted download leaves a cached snapshot without some weights; loading it offline must
+        # surface `huggingface_hub`'s incomplete-snapshot error instead of failing later at model load time.
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            cached_folder = DiffusionPipeline.download(
+                "hf-internal-testing/tiny-stable-diffusion-torch", cache_dir=tmpdirname
+            )
+            for weights_file in Path(cached_folder).glob("unet/diffusion_pytorch_model*"):
+                weights_file.unlink()
+
+            with self.assertRaisesRegex(OSError, "incomplete"):
+                DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch",
+                    cache_dir=tmpdirname,
+                    local_files_only=True,
+                )
+
     def test_download_from_variant_folder(self):
         for use_safetensors in [False, True]:
             other_format = ".bin" if use_safetensors else ".safetensors"