[None][test] Explicitly invalidate HF weight cache on grouped-test teardown

The per-worker HF raw-weight cache previously relied on the shared session's
worker processes exiting to be cleared. Make invalidation explicit: on
shared_llm_4gpu teardown (which runs before the session shuts down its
workers), submit clear_worker_weight_cache() to each worker. This closes the
'setup cache / teardown invalidate' loop instead of leaning on process death.

Signed-off-by: qgai <qgai@nvidia.com>

Author: sunnyqgg

tensorrt_llm/llmapi/_grouped_test_utils.py

@@ -88,6 +88,19 @@ def reset_worker_torch_compile_state() -> None:
     torch._dynamo.reset()
 
 
+def clear_worker_weight_cache() -> None:
+    """Drop the per-worker HF raw-weight cache (runs inside each worker).
+
+    The cache is a process-global keyed by checkpoint file fingerprints, so it
+    otherwise lives until the worker process exits. Submit this to each worker
+    via ``mpi_session.submit_sync(...)`` on group teardown to invalidate it
+    explicitly instead of relying on process death.
+    """
+    from tensorrt_llm._torch.models.checkpoints import HfWeightLoader
+
+    HfWeightLoader.clear_weight_cache()
+
+
 def make_shared_llm(mpi_session):
     """Return an ``LLM`` factory that transparently injects a shared MPI session.
 

tests/integration/defs/accuracy/test_llm_api_pytorch.py

@@ -75,7 +75,9 @@ def patched_start_mpi_pool(self):
 
 
 from tensorrt_llm.llmapi._grouped_test_utils import \
-    hf_weight_cache_env as _hf_weight_cache_env  # noqa: E402
+    clear_worker_weight_cache as _clear_worker_weight_cache  # noqa: E402
+from tensorrt_llm.llmapi._grouped_test_utils import \
+    hf_weight_cache_env as _hf_weight_cache_env
 from tensorrt_llm.llmapi._grouped_test_utils import \
     make_shared_llm as _make_shared_llm
 from tensorrt_llm.llmapi._grouped_test_utils import \
@@ -108,7 +110,14 @@ def shared_mpi_session_4gpu(hf_weight_cache):
 
 @pytest.fixture(scope="module")
 def shared_llm_4gpu(shared_mpi_session_4gpu):
-    return _make_shared_llm(shared_mpi_session_4gpu)
+    try:
+        yield _make_shared_llm(shared_mpi_session_4gpu)
+    finally:
+        # Explicitly invalidate the per-worker HF weight cache while the shared
+        # session is still alive (this fixture tears down before
+        # shared_mpi_session_4gpu), instead of relying on worker process exit.
+        if shared_mpi_session_4gpu is not None:
+            shared_mpi_session_4gpu.submit_sync(_clear_worker_weight_cache)
 
 
 def _get_default_torch_compile_config(torch_compile):