Walk transformers package via importlib.resources at lazy-import time

Author: rasmi

src/transformers/utils/import_utils.py

@@ -18,10 +18,12 @@
 import functools
 import importlib.machinery
 import importlib.metadata
+import importlib.resources
 import importlib.util
 import json
 import operator
 import os
+import pathlib
 import re
 import shutil
 import subprocess
@@ -2671,6 +2673,25 @@ def fetch__all__(file_content) -> list[str]:
         return _all
 
 
+def _resolve_traversable(module_path):
+    """Return a Traversable for `module_path`."""
+    transformers_module = sys.modules.get("transformers")
+    if transformers_module is not None and getattr(transformers_module, "__file__", None) is not None:
+        pkg_root = pathlib.Path(transformers_module.__file__).parent
+        try:
+            rel = pathlib.PurePath(module_path).relative_to(pkg_root)
+        except ValueError:
+            pass
+        else:
+            traversable = importlib.resources.files("transformers")
+            for part in rel.parts:
+                traversable = traversable.joinpath(part)
+            return traversable
+
+    # Fall back to pathlib.Path for paths outside the transformers package.
+    return pathlib.Path(module_path)
+
+
 @lru_cache
 def create_import_structure_from_path(module_path):
     """
@@ -2724,36 +2745,39 @@ def create_import_structure_from_path(module_path):
         }
     }
     """
-    import_structure = {}
-
-    if os.path.isfile(module_path):
+    module_path = str(module_path)
+    if module_path.endswith(".py"):
         module_path = os.path.dirname(module_path)
+    return _create_import_structure_from_traversable(_resolve_traversable(module_path))
 
-    adjacent_modules = []
 
-    with os.scandir(module_path) as entries:
-        for entry in entries:
-            if entry.name == "__pycache__":
-                continue
-            if entry.is_dir():
-                import_structure[entry.name] = create_import_structure_from_path(entry.path)
-            elif not entry.name.startswith(("convert_", "modular_")):
-                adjacent_modules.append(entry.name)
+def _create_import_structure_from_traversable(traversable):
+    """Walk a Traversable and build the import structure."""
+    import_structure = {}
+    adjacent_entries = []
+
+    for entry in traversable.iterdir():
+        if entry.name == "__pycache__":
+            continue
+        if entry.is_dir():
+            # Recurse via the Traversable, not `files(subpackage)` — the latter would force-import the subpackage.
+            import_structure[entry.name] = _create_import_structure_from_traversable(entry)
+        elif not entry.name.startswith(("convert_", "modular_")):
+            adjacent_entries.append(entry)
 
     # We're only taking a look at files different from __init__.py
     # We could theoretically require things directly from the __init__.py
     # files, but this is not supported at this time.
-    if "__init__.py" in adjacent_modules:
-        adjacent_modules.remove("__init__.py")
+    adjacent_entries = [e for e in adjacent_entries if e.name != "__init__.py"]
 
     module_requirements = {}
-    for module_name in adjacent_modules:
+    for entry in adjacent_entries:
         # Only modules ending in `.py` are accepted here.
-        if not module_name.endswith(".py"):
+        if not entry.name.endswith(".py"):
             continue
 
-        with open(os.path.join(module_path, module_name), encoding="utf-8") as f:
-            file_content = f.read()
+        module_name = entry.name
+        file_content = entry.read_text(encoding="utf-8")
 
         # Remove the .py suffix
         module_name = module_name[:-3]