Merge pull request #9 from silveroxides/fix/loading-fixes

Fix models getting stuck in memory when merging

Author: silveroxides

nodes/lora_merger.py

@@ -48,7 +48,7 @@ def merge_multi_loras(
 
     prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
 
-    handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths]
+    handlers = [MemoryEfficientSafeOpen(p, low_memory=True) for p in lora_paths]
 
     try:
         # 1. Analyze all LoRAs
@@ -190,6 +190,12 @@ def merge_multi_loras(
                 del merged_down, merged_up
                 for d, _ in downs: del d
                 for u, _ in ups: del u
+                downs.clear()
+                ups.clear()
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
 
                 pbar.update(1)
 
@@ -412,7 +418,7 @@ def merge_multi_loras_dare(
         print(f"[LoRA Multi-Merge DARE] Drop rate: {drop_rate}, Trim quantile: {trim_quantile}")
 
     prepare_for_large_operation(total_size_gb * 2.5, torch.device(device))
-    handlers = [MemoryEfficientSafeOpen(p) for p in lora_paths]
+    handlers = [MemoryEfficientSafeOpen(p, low_memory=True) for p in lora_paths]
 
     rng = torch.Generator(device=device).manual_seed(seed)
 
@@ -523,6 +529,16 @@ def process_ties_dare(tensors, weights, dim_to_pad):
                 output_sd[f"{core}.lora_up.weight"] = merged_up.to(save_dtype).cpu().contiguous()
                 output_sd[f"{core}.alpha"] = torch.tensor(float(max_rank), dtype=save_dtype)
 
+                del merged_down, merged_up
+                for d, _ in downs: del d
+                for u, _ in ups: del u
+                downs.clear()
+                ups.clear()
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
                 pbar.update(1)
 
         # Final Summary

nodes/lora_resize.py

@@ -461,6 +461,12 @@ def resize_lora_file(
                 if alpha_suffix:
                     output_sd[f"{new_block_name}{alpha_suffix}"] = torch.tensor(result["new_alpha"]).to(save_dtype)
 
+                del result
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
+
                 pbar.update(1)
 
         if verbose and fro_list:
@@ -854,6 +860,10 @@ def extract_core_layer_lora(block_name: str) -> str:
                     del cpu_base
                     stats["copied"] += 1
 
+                import gc
+                gc.collect()
+                if torch.cuda.is_available():
+                    torch.cuda.empty_cache()
 
                 pbar.update(1)
 

nodes/merger.py

@@ -77,18 +77,17 @@ def execute_merge(model_names, calc_mode, all_modes, recipe_params, model_type):
             print(f"[Merger] Preparing memory for {total_size_gb:.2f}GB merge operation...")
             prepare_for_large_operation(total_size_gb * 1.2, torch.device(process_device))
 
-        lazy_load = recipe_params.get('lazy_load', True)
         handlers = {}
         for name in model_names.values():
             if name and name != "None":
                 path = folder_paths.get_full_path(model_type, name)
                 if not path:
                     raise FileNotFoundError(f"Model '{name}' not found.")
-                handlers[name] = MemoryEfficientSafeOpen(path, low_memory=lazy_load)
+                handlers[name] = MemoryEfficientSafeOpen(path, low_memory=True)
 
         primary_handler = handlers[primary_model_name]
         all_keys = primary_handler.keys()
-        metadata = primary_handler.header.get("__metadata__", {})
+        metadata = primary_handler.metadata()
 
         # Convert mismatch_mode string to enum
         mismatch_mode_str = recipe_params.get('mismatch_mode', 'skip')
@@ -125,72 +124,78 @@ def execute_merge(model_names, calc_mode, all_modes, recipe_params, model_type):
         discarded_keys = 0
         error_keys = []
 
-        for key in tqdm(all_keys, desc="Merging layers", unit="layers"):
-            # Check discard patterns first - skip entirely
-            if _matches_any_pattern(key, discard_patterns):
-                discarded_keys += 1
-                pbar.update(1)
-                continue
-
-            # Pre-load Model A's tensor with pinned memory for CUDA
-            cpu_tensor = primary_handler.get_tensor(key)
-            if process_device == 'cuda':
-                tensor_a = transfer_to_gpu_pinned(cpu_tensor, process_device, process_dtype)
-            else:
-                tensor_a = cpu_tensor.to(device=process_device, dtype=process_dtype)
-            del cpu_tensor
-
-            # Check exclude patterns - use Model A only, no merge
-            if _matches_any_pattern(key, exclude_patterns):
-                merged_state_dict[key] = tensor_a.to(save_torch_dtype).cpu().clone()
-                excluded_keys += 1
+        with torch.no_grad():
+            for key in tqdm(all_keys, desc="Merging layers", unit="layers"):
+                # Check discard patterns first - skip entirely
+                if _matches_any_pattern(key, discard_patterns):
+                    discarded_keys += 1
+                    pbar.update(1)
+                    continue
+
+                # Pre-load Model A's tensor with pinned memory for CUDA
+                cpu_tensor = primary_handler.get_tensor(key)
+                if process_device == 'cuda':
+                    tensor_a = transfer_to_gpu_pinned(cpu_tensor, process_device, process_dtype)
+                else:
+                    tensor_a = cpu_tensor.to(device=process_device, dtype=process_dtype)
+                del cpu_tensor
+
+                # Check exclude patterns - use Model A only, no merge
+                if _matches_any_pattern(key, exclude_patterns):
+                    merged_state_dict[key] = tensor_a.detach().to(save_torch_dtype).cpu().clone()
+                    excluded_keys += 1
+                    pbar.update(1)
+                    continue
+
+                # Pass tensor_a metadata to recipes for zeros mode and fallback
+                recipe_params['_tensor_a'] = tensor_a
+                recipe_params['_tensor_a_shape'] = tensor_a.shape
+                recipe_params['_tensor_a_dtype'] = tensor_a.dtype
+
+                try:
+                    recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
+                    result = recipe.merge()
+                except MissingTensorError as e:
+                    if mismatch_mode == MissingTensorBehavior.ERROR:
+                        raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}")
+                    result = None
+                    error_keys.append(key)
+
+                # Handle None result (mismatch occurred with skip mode)
+                if result is None:
+                    result = tensor_a
+                    skipped_keys += 1
+
+                if isinstance(result, dict):
+                    for r_key, r_tensor in result.items():
+                        merged_state_dict[r_key] = r_tensor.detach().to(save_torch_dtype).cpu().clone()
+                else:
+                    # Ensure compatibility with Model A's architecture.
+                    # If alignment_mode is 'pad/crop', we crop results that were padded.
+                    # If alignment_mode is 'interpolate', resizing happened during operators.
+                    if alignment_mode == 'pad/crop':
+                        target_shape = recipe_params['_tensor_a_shape']
+                        if result.shape != target_shape:
+                            slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(result.shape, target_shape))
+                            result = result[slices]
+
+                    merged_state_dict[key] = result.detach().to(save_torch_dtype).cpu().clone()
+
+                # Clean up references to allow GC immediately.
+                # Local loop variables must be explicitly deleted to prevent PyTorch from keeping tensors in VRAM.
+                recipe.clean()
+                del recipe_params['_tensor_a']
+                del tensor_a
+                del recipe
+                del result
+
+                if recipe_params.get('force_clear_cache', True):
+                    import gc
+                    gc.collect()
+                    if torch.cuda.is_available():
+                        torch.cuda.empty_cache()
+
                 pbar.update(1)
-                continue
-
-            # Pass tensor_a metadata to recipes for zeros mode and fallback
-            recipe_params['_tensor_a'] = tensor_a
-            recipe_params['_tensor_a_shape'] = tensor_a.shape
-            recipe_params['_tensor_a_dtype'] = tensor_a.dtype
-
-            try:
-                recipe = calc_mode_class.create_recipe(key=key, **recipe_params)
-                result = recipe.merge()
-            except MissingTensorError as e:
-                if mismatch_mode == MissingTensorBehavior.ERROR:
-                    raise ValueError(f"Layer mismatch error (mismatch_mode='error'): {e}")
-                result = None
-                error_keys.append(key)
-
-            # Handle None result (mismatch occurred with skip mode)
-            if result is None:
-                result = tensor_a
-                skipped_keys += 1
-
-            if isinstance(result, dict):
-                for r_key, r_tensor in result.items():
-                    merged_state_dict[r_key] = r_tensor.to(save_torch_dtype).cpu().clone()
-            else:
-                # Ensure compatibility with Model A's architecture.
-                # If alignment_mode is 'pad/crop', we crop results that were padded.
-                # If alignment_mode is 'interpolate', resizing happened during operators.
-                if alignment_mode == 'pad/crop':
-                    target_shape = recipe_params['_tensor_a_shape']
-                    if result.shape != target_shape:
-                        slices = tuple(slice(0, min(res_s, tgt_s)) for res_s, tgt_s in zip(result.shape, target_shape))
-                        result = result[slices]
-
-                merged_state_dict[key] = result.to(save_torch_dtype).cpu().clone()
-
-            # Clean up tensor_a reference to allow GC
-            del recipe_params['_tensor_a']
-
-            if recipe_params.get('force_clear_cache', True):
-                import gc
-                gc.collect()
-                if torch.cuda.is_available():
-                    torch.cuda.empty_cache()
-
-            pbar.update(1)
 
         # Log summary
         if excluded_keys > 0:

nodes/merger_ops.py

@@ -4,6 +4,8 @@
 from enum import Enum
 
 
+from .merger_utils import transfer_to_gpu_pinned
+
 class MissingTensorBehavior(Enum):
     """Controls behavior when a tensor key is missing from a model."""
     ERROR = "error"      # Raise exception (strict mode)
@@ -127,12 +129,21 @@ def oper(self, *args) -> torch.Tensor:
         raise NotImplementedError
 
     def recurse(self, operation):
-        source_tensors = [source_oper.merge() for source_oper in operation.sources]
-        return operation.oper(*source_tensors)
+        self._source_tensors = [source_oper.merge() for source_oper in operation.sources]
+        return operation.oper(*self._source_tensors)
 
     def merge(self):
         return self.merge_func(self)
 
+    def clean(self):
+        if hasattr(self, '_source_tensors'):
+            for t in self._source_tensors:
+                del t
+            del self._source_tensors
+        for source in self.sources:
+            if hasattr(source, 'clean'):
+                source.clean()
+
 class LoadTensor(Operation):
     def __init__(self, key, model_name, handlers, device, dtype,
                  on_missing=MissingTensorBehavior.ERROR, fallback_shape=None, fallback_dtype=None):
@@ -165,7 +176,17 @@ def merge(self) -> torch.Tensor:
                 dtype = self.fallback_dtype if self.fallback_dtype else self.dtype
                 return torch.zeros(self.fallback_shape, device=self.device, dtype=dtype)
 
-        return handler.get_tensor(self.key).to(device=self.device, dtype=self.dtype)
+        cpu_tensor = handler.get_tensor(self.key)
+        if self.device == 'cuda':
+            self._tensor = transfer_to_gpu_pinned(cpu_tensor, self.device, self.dtype)
+        else:
+            self._tensor = cpu_tensor.to(device=self.device, dtype=self.dtype)
+        del cpu_tensor
+        return self._tensor
+
+    def clean(self):
+        if hasattr(self, '_tensor'):
+            del self._tensor
 
 class Multiply(Operation):
     def __init__(self, key, alpha, *sources):
@@ -409,6 +430,10 @@ def __init__(self, key, tensor):
     def merge(self) -> torch.Tensor:
         return self.tensor
 
+    def clean(self):
+        if hasattr(self, 'tensor'):
+            del self.tensor
+
 class WeightSum(CalcMode):
     name = 'Weight-Sum'
     description = 'A * (1 - α) + B * α. Simple linear interpolation.'