[Fix] Fix hf cache monkey patch failure when HF_CACHE set to shared storage (#1673)

[Fix] Fix `monkey_patch_hf_modules_cache` failure when HF_CACHE set to shared storage

Author: nil0x9

xtuner/v1/utils/misc.py

@@ -1,4 +1,5 @@
 import os
+import socket
 import sys
 import threading
 from functools import reduce
@@ -19,7 +20,7 @@
 from .logger import get_logger
 
 
-HF_PATCH_MODULES_CACHE_PREFIX = "modules_pid_"
+HF_PATCH_MODULES_CACHE_PREFIX = "modules_cache"
 
 logger = get_logger()
 XTUNER_DETERMINISTIC = os.getenv("XTUNER_DETERMINISTIC") == "true"
@@ -134,18 +135,21 @@ def is_hf_model_path(path: str | Path) -> tuple[bool, Exception | None]:
 
 
 def monkey_patch_hf_modules_cache():
-    # 如果在hf中tokenizer、config等使用remote_code，例如 `AutoConfig.from_pretrained(hf_model_path, trust_remote_code=True)`，
-    # 会将hf_model_path 拷贝到 HF_MODULES_CACHE 中。 如果单机八卡机器上多个进程同时读写此目录，会导致冲突。
-    # 因此需要将 HF_MODULES_CACHE 设置为当前进程的临时目录。
-    modules_cache = os.path.join(constants.HF_HOME, f"{HF_PATCH_MODULES_CACHE_PREFIX}{os.getpid()}")
+    # When using remote_code in HF for tokenizer, config, etc., e.g., `AutoConfig.from_pretrained(hf_model_path,
+    # trust_remote_code=True)`, the hf_model_path will be copied to HF_MODULES_CACHE. If multiple processes read/write
+    # this directory simultaneously, it will cause conflicts. Therefore, we need to set HF_MODULES_CACHE to a unique
+    # temporary directory (identified w/ hostname + pid) for the current process.
+    hostname = socket.gethostname()
+    pid = os.getpid()
+    modules_cache = os.path.join(constants.HF_HOME, f"{HF_PATCH_MODULES_CACHE_PREFIX}_{hostname}_{pid}")
     os.environ["HF_MODULES_CACHE"] = modules_cache
     transformers.utils.hub.HF_MODULES_CACHE = modules_cache
-    # 在 import 时刻，Python 会在 dynamic_module_utils 模块的命名空间中创建一个新的名字 HF_MODULES_CACHE，
-    # 并将其绑定到 transformers.utils.HF_MODULES_CACHE 当时所指向的对象。
-    # 因此，需要将 transformers.dynamic_module_utils.HF_MODULES_CACHE 也设置为新的 modules_cache。
+    # At import time, Python creates a new name HF_MODULES_CACHE in the dynamic_module_utils module's namespace,
+    # binding it to the object that transformers.utils.HF_MODULES_CACHE pointed to at that moment.
+    # Therefore, we need to set transformers.dynamic_module_utils.HF_MODULES_CACHE to the new modules_cache as well.
     transformers.dynamic_module_utils.HF_MODULES_CACHE = modules_cache
     transformers.utils.HF_MODULES_CACHE = modules_cache
-    logger.info(f"set HF_MODULES_CACHE to {modules_cache} for current process {os.getpid()}")
+    logger.info(f"set HF_MODULES_CACHE to {modules_cache} for current process (hostname={hostname}, pid={pid})")
 
 
 class FunctionEnum(StrEnum):