Fix duplicate safetensors.load_file call in _onload_from_disk when st… (#13851)

Fix duplicate safetensors.load_file call in _onload_from_disk when stream is None

Signed-off-by: Gagan Dhakrey <gagandhakrey@gmail.com>
Co-authored-by: Sayak Paul <spsayakpaul@gmail.com>

Author: gagandhakrey

src/diffusers/hooks/group_offloading.py

@@ -259,17 +259,16 @@ def _onload_from_disk(self):
         current_stream = self._torch_accelerator_module.current_stream() if self.record_stream else None
 
         with context:
-            # Load to CPU (if using streams) or directly to target device, pin, and async copy to device
-            device = str(self.onload_device) if self.stream is None else "cpu"
-            loaded_tensors = safetensors.torch.load_file(self.safetensors_file_path, device=device)
-
             if self.stream is not None:
+                # Load to CPU first, pin memory, then async copy to the target device
+                loaded_tensors = safetensors.torch.load_file(self.safetensors_file_path, device="cpu")
                 for key, tensor_obj in self.key_to_tensor.items():
                     pinned_tensor = loaded_tensors[key].pin_memory()
                     tensor_obj.data = pinned_tensor.to(self.onload_device, non_blocking=self.non_blocking)
                     if self.record_stream:
                         tensor_obj.data.record_stream(current_stream)
             else:
+                # Load directly to the target device
                 onload_device = (
                     self.onload_device.type if isinstance(self.onload_device, torch.device) else self.onload_device
                 )