align snapshot_download to respect hfh latest version

src/diffusers/pipelines/pipeline_utils.py

@@ -38,7 +38,12 @@
     read_dduf_file,
     snapshot_download,
 )
-from huggingface_hub.utils import HfHubHTTPError, OfflineModeIsEnabled, validate_hf_hub_args
+from huggingface_hub.utils import (
+    HfHubHTTPError,
+    LocalEntryNotFoundError,
+    OfflineModeIsEnabled,
+    validate_hf_hub_args,
+)
 from packaging import version
 from tqdm.auto import tqdm
 from typing_extensions import Self
@@ -1657,10 +1662,37 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
                 force_download=force_download,
                 token=token,
             )
+            filenames = {sibling.rfilename for sibling in info.siblings}
+        else:
+            # We are offline (either the user asked for `local_files_only` or the `model_info` call above
+            # failed), so the repo's file listing needed to compute the allow/ignore patterns below cannot be
+            # fetched. Source the listing from the cached snapshot instead: it holds exactly the (already
+            # filtered) files a previous online call downloaded, so both code paths compute the same patterns
+            # and `snapshot_download` validates the cached snapshot against them (catching e.g. a download
+            # that was interrupted midway). If the config isn't cached, skip straight to `snapshot_download`
+            # below, which raises the appropriate offline not-found error.
+            try:
+                config_file = hf_hub_download(
+                    pretrained_model_name,
+                    cls.config_name,
+                    cache_dir=cache_dir,
+                    revision=revision,
+                    token=token,
+                    local_files_only=True,
+                )
+            except LocalEntryNotFoundError:
+                config_file = None
+
+        if config_file is not None:
+            snapshot_folder = Path(config_file).parent
+            if local_files_only:
+                filenames = {
+                    f.relative_to(snapshot_folder).as_posix() for f in snapshot_folder.rglob("*") if f.is_file()
+                }
+
             config_dict = cls._dict_from_json_file(config_file)
             ignore_filenames = config_dict.pop("_ignore_files", [])
 
-            filenames = {sibling.rfilename for sibling in info.siblings}
             if variant is not None and _check_legacy_sharding_variant_format(filenames=filenames, variant=variant):
                 warn_msg = (
                     f"Warning: The repository contains sharded checkpoints for variant '{variant}' maybe in a deprecated format. "
@@ -1675,9 +1707,11 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
                 logger.warning(warn_msg)
 
             filenames = set(filenames) - set(ignore_filenames)
-            if revision in DEPRECATED_REVISION_ARGS and version.parse(
-                version.parse(__version__).base_version
-            ) >= version.parse("0.22.0"):
+            if (
+                not local_files_only
+                and revision in DEPRECATED_REVISION_ARGS
+                and version.parse(version.parse(__version__).base_version) >= version.parse("0.22.0")
+            ):
                 warn_deprecated_model_variant(pretrained_model_name, token, variant, revision, filenames)
 
             custom_components, folder_names = _get_custom_components_and_folders(
@@ -1762,19 +1796,20 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
 
             # Don't download index files of forbidden patterns either
             ignore_patterns = ignore_patterns + [f"{i}.index.*json" for i in ignore_patterns]
-            re_ignore_pattern = [re.compile(fnmatch.translate(p)) for p in ignore_patterns]
-            re_allow_pattern = [re.compile(fnmatch.translate(p)) for p in allow_patterns]
 
-            expected_files = [f for f in filenames if not any(p.match(f) for p in re_ignore_pattern)]
-            expected_files = [f for f in expected_files if any(p.match(f) for p in re_allow_pattern)]
+            if not local_files_only:
+                re_ignore_pattern = [re.compile(fnmatch.translate(p)) for p in ignore_patterns]
+                re_allow_pattern = [re.compile(fnmatch.translate(p)) for p in allow_patterns]
 
-            snapshot_folder = Path(config_file).parent
-            pipeline_is_cached = all((snapshot_folder / f).is_file() for f in expected_files)
+                expected_files = [f for f in filenames if not any(p.match(f) for p in re_ignore_pattern)]
+                expected_files = [f for f in expected_files if any(p.match(f) for p in re_allow_pattern)]
+
+                pipeline_is_cached = all((snapshot_folder / f).is_file() for f in expected_files)
 
-            if pipeline_is_cached and not force_download:
-                # if the pipeline is cached, we can directly return it
-                # else call snapshot_download
-                return snapshot_folder
+                if pipeline_is_cached and not force_download:
+                    # if the pipeline is cached, we can directly return it
+                    # else call snapshot_download
+                    return snapshot_folder
 
         user_agent = {"pipeline_class": cls.__name__}
         if custom_pipeline is not None and not custom_pipeline.endswith(".py"):
@@ -1817,7 +1852,7 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
 
             return cached_folder
 
-        except FileNotFoundError:
+        except FileNotFoundError as e:
             # Means we tried to load pipeline with `local_files_only=True` but the files have not been found in local cache.
             # This can happen in two cases:
             # 1. If the user passed `local_files_only=True`                    => we raise the error directly
@@ -1828,9 +1863,9 @@ def download(cls, pretrained_model_name, **kwargs) -> str | os.PathLike:
             else:
                 # 2. we forced `local_files_only=True` when `model_info` failed
                 raise EnvironmentError(
-                    f"Cannot load model {pretrained_model_name}: model is not cached locally and an error occurred"
-                    " while trying to fetch metadata from the Hub. Please check out the root cause in the stacktrace"
-                    " above."
+                    f"Cannot load model {pretrained_model_name}: the model is not fully cached locally ({e}) and an"
+                    " error occurred while trying to fetch metadata from the Hub. Please check out the root cause in"
+                    " the stacktrace above."
                 ) from model_info_call_error
 
     @classmethod

tests/models/test_modeling_common.py

@@ -161,9 +161,11 @@ def test_local_files_only_with_sharded_checkpoint(self):
                     repo_id, subfolder="transformer", cache_dir=tmpdir, local_files_only=True
                 )
 
-            # Verify error mentions the missing shard
+            # Verify error mentions the missing shard. `huggingface_hub>=1.22.0` reports it via
+            # `IncompleteSnapshotError` (repo-relative path), older versions via diffusers' own check on the
+            # cached folder (absolute path), so match on the shard's filename.
             error_msg = str(context.value)
-            assert cached_shard_file in error_msg or "required according to the checkpoint index" in error_msg, (
+            assert os.path.basename(cached_shard_file) in error_msg, (
                 f"Expected error about missing shard, got: {error_msg}"
             )
 

tests/pipelines/test_pipelines.py

@@ -25,6 +25,7 @@
 import unittest
 import unittest.mock as mock
 import warnings
+from pathlib import Path
 
 import numpy as np
 import PIL.Image
@@ -504,6 +505,51 @@ def test_local_files_only_are_used_when_no_internet(self):
                 if p1.data.ne(p2.data).sum() > 0:
                     assert False, "Parameters not the same!"
 
+    def test_local_files_only_uses_same_snapshot_download_patterns(self):
+        # diffusers downloads a filtered subset of a repo (skipping e.g. `.gitattributes`). Newer
+        # `huggingface_hub` versions validate that a cached snapshot contains every file matching the
+        # requested patterns under `local_files_only=True`, so an offline
+        # `snapshot_download(allow_patterns=None)` would wrongly expect the whole repo. The offline path
+        # must compute the same patterns as the online one (from the cached snapshot instead of
+        # `model_info`). See https://github.com/huggingface/diffusers/issues/14117
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with mock.patch(
+                "diffusers.pipelines.pipeline_utils.snapshot_download", side_effect=snapshot_download
+            ) as mock_snapshot_download:
+                cached_folder = DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch", cache_dir=tmpdirname
+                )
+                online_kwargs = mock_snapshot_download.call_args.kwargs
+
+                offline_folder = DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch",
+                    cache_dir=tmpdirname,
+                    local_files_only=True,
+                )
+                offline_kwargs = mock_snapshot_download.call_args.kwargs
+
+            assert os.path.samefile(offline_folder, cached_folder)
+            assert set(offline_kwargs["allow_patterns"]) == set(online_kwargs["allow_patterns"])
+            assert set(offline_kwargs["ignore_patterns"]) == set(online_kwargs["ignore_patterns"])
+
+    @require_hf_hub_version_greater("1.21.0")
+    def test_local_files_only_raises_for_snapshot_with_missing_weights(self):
+        # An interrupted download leaves a cached snapshot without some weights; loading it offline must
+        # surface `huggingface_hub`'s incomplete-snapshot error instead of failing later at model load time.
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            cached_folder = DiffusionPipeline.download(
+                "hf-internal-testing/tiny-stable-diffusion-torch", cache_dir=tmpdirname
+            )
+            for weights_file in Path(cached_folder).glob("unet/diffusion_pytorch_model*"):
+                weights_file.unlink()
+
+            with self.assertRaisesRegex(OSError, "incomplete"):
+                DiffusionPipeline.download(
+                    "hf-internal-testing/tiny-stable-diffusion-torch",
+                    cache_dir=tmpdirname,
+                    local_files_only=True,
+                )
+
     def test_download_from_variant_folder(self):
         for use_safetensors in [False, True]:
             other_format = ".bin" if use_safetensors else ".safetensors"