Merge branch 'mergeability-pr-45541' into all-defects

Author: evalstate

src/transformers/tokenization_utils_base.py

@@ -1705,6 +1705,13 @@ def from_pretrained(
                 else:
                     vocab_files["vocab_file"] = match.group()
 
+        error_message = (
+            f"Can't load tokenizer for '{pretrained_model_name_or_path}'. If you were trying to load it from "
+            "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
+            f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory "
+            f"containing all relevant files for a {cls.__name__} tokenizer."
+        )
+
         resolved_vocab_files = {}
         for file_id, file_path in vocab_files.items():
             if file_path is None:
@@ -1733,17 +1740,19 @@ def from_pretrained(
                     raise
                 except Exception:
                     # For any other exception, we throw a generic error.
-                    raise OSError(
-                        f"Can't load tokenizer for '{pretrained_model_name_or_path}'. If you were trying to load it from "
-                        "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
-                        f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory "
-                        f"containing all relevant files for a {cls.__name__} tokenizer."
-                    )
+                    raise OSError(error_message)
                 commit_hash = extract_commit_hash(resolved_vocab_files[file_id], commit_hash)
 
-        for file_id, file_path in vocab_files.items():
-            if file_id not in resolved_vocab_files:
-                continue
+        loadable_file_ids = set(cls.vocab_files_names)
+        if loadable_file_ids and "tokenizer_file" in resolved_vocab_files:
+            loadable_file_ids.add("tokenizer_file")
+        loadable_file_ids.intersection_update(resolved_vocab_files)
+        if (
+            (local_files_only or is_local)
+            and loadable_file_ids
+            and all(resolved_vocab_files[file_id] is None for file_id in loadable_file_ids)
+        ):
+            raise OSError(error_message)
 
         return cls._from_pretrained(
             resolved_vocab_files,