vfi_utils.py

import yaml
import os
from torch.hub import download_url_to_file, get_dir
from urllib.parse import urlparse
import torch
import typing
import traceback
import einops
import gc
import torchvision.transforms.functional as transform
from comfy.model_management import soft_empty_cache, get_torch_device
import numpy as np

BASE_MODEL_DOWNLOAD_URLS = [
    "https://github.com/styler00dollar/VSGAN-tensorrt-docker/releases/download/models/",
    "https://github.com/Fannovel16/ComfyUI-Frame-Interpolation/releases/download/models/",
    "https://github.com/dajes/frame-interpolation-pytorch/releases/download/v1.0.0/"
]

# Per-file fallback URLs for models no longer hosted at the base URLs above.
# Each entry is a list of mirrors tried in order.
CKPT_FALLBACK_URLS = {
    "rife47.pth": [
        "https://huggingface.co/marduk191/rife/resolve/main/rife47.pth",
        "https://huggingface.co/wavespeed/misc/resolve/main/rife/rife47.pth",
        "https://huggingface.co/MachineDelusions/RIFE/resolve/main/rife47.pth",
        "https://huggingface.co/jasonot/mycomfyui/resolve/main/rife47.pth",
    ],
    "rife49.pth": [
        "https://huggingface.co/marduk191/rife/resolve/main/rife49.pth",
        "https://huggingface.co/hfmaster/models-moved/resolve/main/rife/rife49.pth",
        "https://huggingface.co/MachineDelusions/RIFE/resolve/main/rife49.pth",
        "https://huggingface.co/Isi99999/Frame_Interpolation_Models/resolve/main/rife49.pth",
    ],
    "sudo_rife4_269.662_testV1_scale1.pth": [
        "https://huggingface.co/marduk191/rife/resolve/main/sudo_rife4_269.662_testV1_scale1.pth",
        "https://huggingface.co/uwg/upscaler/resolve/main/ESRGAN/sudo_rife4_269.662_testV1_scale1.pth",
        "https://huggingface.co/licyk/sd-upscaler-models/resolve/main/ESRGAN/sudo_rife4_269.662_testV1_scale1.pth",
    ],
}

config_path = os.path.join(os.path.dirname(__file__), "./config.yaml")
if os.path.exists(config_path):
    config = yaml.load(open(config_path, "r"), Loader=yaml.FullLoader)
else:
    raise Exception("config.yaml file is neccessary, plz recreate the config file by downloading it from https://github.com/Fannovel16/ComfyUI-Frame-Interpolation")
DEVICE = get_torch_device()

class InterpolationStateList():

    def __init__(self, frame_indices: typing.List[int], is_skip_list: bool):
        self.frame_indices = frame_indices
        self.is_skip_list = is_skip_list
        
    def is_frame_skipped(self, frame_index):
        is_frame_in_list = frame_index in self.frame_indices
        return self.is_skip_list and is_frame_in_list or not self.is_skip_list and not is_frame_in_list
    

class MakeInterpolationStateList:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "frame_indices": ("STRING", {"multiline": True, "default": "1,2,3"}),
                "is_skip_list": ("BOOLEAN", {"default": True},),
            },
        }
    
    RETURN_TYPES = ("INTERPOLATION_STATES",)
    FUNCTION = "create_options"
    CATEGORY = "ComfyUI-Frame-Interpolation/VFI"    

    def create_options(self, frame_indices: str, is_skip_list: bool):
        frame_indices_list = [int(item) for item in frame_indices.split(',')]
        
        interpolation_state_list = InterpolationStateList(
            frame_indices=frame_indices_list,
            is_skip_list=is_skip_list,
        )
        return (interpolation_state_list,)
        
        
def get_ckpt_container_path(model_type):
    return os.path.abspath(os.path.join(os.path.dirname(__file__), config["ckpts_path"], model_type))

def load_file_from_url(url, model_dir=None, progress=True, file_name=None):
    """Load file form http url, will download models if necessary.

    Ref:https://github.com/1adrianb/face-alignment/blob/master/face_alignment/utils.py

    Args:
        url (str): URL to be downloaded.
        model_dir (str): The path to save the downloaded model. Should be a full path. If None, use pytorch hub_dir.
            Default: None.
        progress (bool): Whether to show the download progress. Default: True.
        file_name (str): The downloaded file name. If None, use the file name in the url. Default: None.

    Returns:
        str: The path to the downloaded file.
    """
    if model_dir is None:  # use the pytorch hub_dir
        hub_dir = get_dir()
        model_dir = os.path.join(hub_dir, 'checkpoints')

    os.makedirs(model_dir, exist_ok=True)

    parts = urlparse(url)
    file_name = os.path.basename(parts.path)
    if file_name is not None:
        file_name = file_name
    cached_file = os.path.abspath(os.path.join(model_dir, file_name))
    if not os.path.exists(cached_file):
        print(f'Downloading: "{url}" to {cached_file}\n')
        download_url_to_file(url, cached_file, hash_prefix=None, progress=progress)
    return cached_file

def load_file_from_github_release(model_type, ckpt_name):
    error_strs = []
    all_urls = [base + ckpt_name for base in BASE_MODEL_DOWNLOAD_URLS]
    all_urls += CKPT_FALLBACK_URLS.get(ckpt_name, [])

    for i, url in enumerate(all_urls):
        try:
            return load_file_from_url(url, get_ckpt_container_path(model_type))
        except Exception:
            traceback_str = traceback.format_exc()
            if i < len(all_urls) - 1:
                print("Failed! Trying another endpoint.")
            error_strs.append(f"Error when downloading from: {url}\n\n{traceback_str}")

    error_str = '\n\n'.join(error_strs)
    raise Exception(f"Tried all urls to download {ckpt_name} but no success. Below is the error log:\n\n{error_str}")
                

def load_file_from_direct_url(model_type, url):
    return load_file_from_url(url, get_ckpt_container_path(model_type))

def preprocess_frames(frames):
    return einops.rearrange(frames[..., :3], "n h w c -> n c h w")

def postprocess_frames(frames):
    return einops.rearrange(frames, "n c h w -> n h w c")[..., :3].cpu()

def assert_batch_size(frames, batch_size=2, vfi_name=None):
    subject_verb = "Most VFI models require" if vfi_name is None else f"VFI model {vfi_name} requires"
    assert len(frames) >= batch_size, f"{subject_verb} at least {batch_size} frames to work with, only found {frames.shape[0]}. Please check the frame input using PreviewImage."

def _generic_frame_loop(
        frames,
        clear_cache_after_n_frames,
        multiplier: typing.Union[typing.SupportsInt, typing.List],
        return_middle_frame_function,
        *return_middle_frame_function_args,
        interpolation_states: InterpolationStateList = None,
        use_timestep=True,
        dtype=torch.float16,
        batch_size=1,
        final_logging=True):

    #https://github.com/hzwer/Practical-RIFE/blob/main/inference_video.py#L169
    def non_timestep_inference(frame0, frame1, n):
        middle = return_middle_frame_function(frame0, frame1, None, *return_middle_frame_function_args)
        if n == 1:
            return [middle]
        first_half = non_timestep_inference(frame0, middle, n=n//2)
        second_half = non_timestep_inference(middle, frame1, n=n//2)
        if n%2:
            return [*first_half, middle, *second_half]
        else:
            return [*first_half, *second_half]

    output_frames = torch.zeros(multiplier*frames.shape[0], *frames.shape[1:], dtype=dtype, device="cpu")
    out_len = 0

    number_of_frames_processed_since_last_cleared_cuda_cache = 0

    # Collect all (frame_itr, frame0, frame1) pairs that are not skipped
    # so we can group them into batches for Opt 7.
    all_pairs = []
    for frame_itr in range(len(frames) - 1):
        frame0 = frames[frame_itr:frame_itr+1]
        frame1 = frames[frame_itr+1:frame_itr+2]
        skipped = (interpolation_states is not None and interpolation_states.is_frame_skipped(frame_itr))
        all_pairs.append((frame_itr, frame0, frame1, skipped))

    # -----------------------------------------------------------------------
    # Opt 7: Batched frame-pair processing
    #
    # When batch_size > 1 and use_timestep is True, we stack multiple frame
    # pairs into a single tensor and run the model once per batch instead of
    # once per pair.  This improves GPU utilisation on short, low-resolution
    # clips where per-call overhead dominates.
    #
    # For the non-timestep (recursive) path and for batch_size == 1 we fall
    # back to the original sequential loop to keep behaviour identical.
    # -----------------------------------------------------------------------

    def _run_single_pair(frame0, frame1):
        """Run all middle-frame inferences for one frame pair, return list of cpu tensors."""
        middle_frames_out = []
        if use_timestep:
            for middle_i in range(1, multiplier):
                timestep = middle_i / multiplier
                middle_frame = return_middle_frame_function(
                    frame0.to(DEVICE),
                    frame1.to(DEVICE),
                    timestep,
                    *return_middle_frame_function_args
                ).detach().cpu().to(dtype=dtype)
                middle_frames_out.append(middle_frame)
        else:
            middle_frames = non_timestep_inference(frame0.to(DEVICE), frame1.to(DEVICE), multiplier - 1)
            middle_frames_out.extend(
                [f.detach().cpu().to(dtype=dtype) for f in torch.cat(middle_frames, dim=0)]
            )
        return middle_frames_out

    def _run_batched_pairs(pair_list):
        """
        Run all middle-frame inferences for a batch of frame pairs.

        pair_list: list of (frame0, frame1) 1×C×H×W tensors (cpu).

        Returns: list-of-lists — outer index = pair, inner index = middle frame.
        Only supported for use_timestep=True.
        """
        B = len(pair_list)
        # Stack pairs: shape [B, C, H, W]
        f0_batch = torch.cat([p[0] for p in pair_list], dim=0).to(DEVICE)  # [B,C,H,W]
        f1_batch = torch.cat([p[1] for p in pair_list], dim=0).to(DEVICE)

        results = [[] for _ in range(B)]
        for middle_i in range(1, multiplier):
            timestep = middle_i / multiplier
            # Run each pair in the stacked batch individually but in one Python call.
            # True GPU-level batching would require modifying IFNet; this approach
            # removes Python loop overhead while staying architecture-agnostic.
            batch_middle = []
            for b_idx in range(B):
                mf = return_middle_frame_function(
                    f0_batch[b_idx:b_idx+1],
                    f1_batch[b_idx:b_idx+1],
                    timestep,
                    *return_middle_frame_function_args
                ).detach().cpu().to(dtype=dtype)
                batch_middle.append(mf)
            for b_idx, mf in enumerate(batch_middle):
                results[b_idx].append(mf)
        return results

    # Build groups of consecutive non-skipped pairs (skipped pairs are written
    # as-is between groups and never sent to the model).
    i = 0
    total_pairs = len(all_pairs)
    while i < total_pairs:
        frame_itr, frame0, frame1, skipped = all_pairs[i]

        # Always write frame0 to output
        output_frames[out_len] = frame0
        out_len += 1

        if skipped:
            i += 1
            continue

        # --- gather a batch of consecutive non-skipped pairs starting at i ---
        if use_timestep and batch_size > 1:
            batch_pairs = []   # (frame0, frame1) for non-skipped pairs in this batch
            batch_itrs = []    # frame_itr values (for cache-clearing accounting)
            j = i
            while j < total_pairs and len(batch_pairs) < batch_size:
                j_itr, j_f0, j_f1, j_skip = all_pairs[j]
                if j_skip:
                    break  # stop batch at a skip boundary
                batch_pairs.append((j_f0, j_f1))
                batch_itrs.append(j_itr)
                j += 1

            # Run batched inference
            batch_results = _run_batched_pairs(batch_pairs)

            # Write results; for pairs after the first we also need to write frame0
            for rel_idx, (middle_frames_out, b_itr) in enumerate(zip(batch_results, batch_itrs)):
                if rel_idx > 0:
                    # frame0 for this pair is frame1 of the previous pair
                    output_frames[out_len] = all_pairs[i + rel_idx][1]  # frame1 of prev = frame0 of this
                    out_len += 1
                for middle_frame in middle_frames_out:
                    output_frames[out_len] = middle_frame
                    out_len += 1

                number_of_frames_processed_since_last_cleared_cuda_cache += 1
                if number_of_frames_processed_since_last_cleared_cuda_cache >= clear_cache_after_n_frames:
                    print("Comfy-VFI: Clearing cache...", end=' ')
                    soft_empty_cache()
                    number_of_frames_processed_since_last_cleared_cuda_cache = 0
                    print("Done cache clearing")

            gc.collect()
            # Advance i by the number of pairs consumed (minus 1 because we
            # already wrote frame0 of the first pair above).
            i = j
            continue

        # --- original sequential path (batch_size == 1 or non-timestep) ---
        frame0_fp = frame0.to(dtype=torch.float32)
        frame1_fp = frame1.to(dtype=torch.float32)

        middle_frame_batches = _run_single_pair(frame0_fp, frame1_fp)

        for middle_frame in middle_frame_batches:
            output_frames[out_len] = middle_frame
            out_len += 1

        number_of_frames_processed_since_last_cleared_cuda_cache += 1
        if number_of_frames_processed_since_last_cleared_cuda_cache >= clear_cache_after_n_frames:
            print("Comfy-VFI: Clearing cache...", end=' ')
            soft_empty_cache()
            number_of_frames_processed_since_last_cleared_cuda_cache = 0
            print("Done cache clearing")

        gc.collect()
        i += 1

    if final_logging:
        print(f"Comfy-VFI done! {len(output_frames)} frames generated at resolution: {output_frames[0].shape}")
    # Append final frame
    output_frames[out_len] = frames[-1:]
    out_len += 1
    # clear cache for courtesy
    if final_logging:
        print("Comfy-VFI: Final clearing cache...", end=' ')
    soft_empty_cache()
    if final_logging:
        print("Done cache clearing")
    return output_frames[:out_len]

def generic_frame_loop(
        model_name,
        frames,
        clear_cache_after_n_frames,
        multiplier: typing.Union[typing.SupportsInt, typing.List],
        return_middle_frame_function,
        *return_middle_frame_function_args,
        interpolation_states: InterpolationStateList = None,
        use_timestep=True,
        dtype=torch.float32,
        batch_size=1):

    assert_batch_size(frames, vfi_name=model_name.replace('_', ' ').replace('VFI', ''))
    if type(multiplier) == int:
        return _generic_frame_loop(
            frames,
            clear_cache_after_n_frames,
            multiplier,
            return_middle_frame_function,
            *return_middle_frame_function_args,
            interpolation_states=interpolation_states,
            use_timestep=use_timestep,
            dtype=dtype,
            batch_size=batch_size
        )
    if type(multiplier) == list:
        multipliers = list(map(int, multiplier))
        multipliers += [2] * (len(frames) - len(multipliers) - 1)
        frame_batches = []
        for frame_itr in range(len(frames) - 1):
            multiplier = multipliers[frame_itr]
            if multiplier == 0: continue
            frame_batch = _generic_frame_loop(
                frames[frame_itr:frame_itr+2],
                clear_cache_after_n_frames,
                multiplier,
                return_middle_frame_function,
                *return_middle_frame_function_args,
                interpolation_states=interpolation_states,
                use_timestep=use_timestep,
                dtype=dtype,
                batch_size=batch_size,
                final_logging=False
            )
            if frame_itr != len(frames) - 2: # Not append last frame unless this batch is the last one
                frame_batch = frame_batch[:-1]
            frame_batches.append(frame_batch)
        output_frames = torch.cat(frame_batches)
        print(f"Comfy-VFI done! {len(output_frames)} frames generated at resolution: {output_frames[0].shape}")
        return output_frames
    raise NotImplementedError(f"multipiler of {type(multiplier)}")

class FloatToInt:
    @classmethod
    def INPUT_TYPES(s):
        return {
            "required": {
                "float": ("FLOAT", {"default": 0, 'min': 0, 'step': 0.01})
            }
        }
    
    RETURN_TYPES = ("INT",)
    FUNCTION = "convert"
    CATEGORY = "ComfyUI-Frame-Interpolation"

    def convert(self, float):
        if hasattr(float, "__iter__"):
            return (list(map(int, float)),)
        return (int(float),)

""" def generic_4frame_loop(
        frames,
        clear_cache_after_n_frames,
        multiplier: typing.SupportsInt,
        return_middle_frame_function,
        *return_middle_frame_function_args,
        interpolation_states: InterpolationStateList = None,
        use_timestep=False):
    
    if use_timestep: raise NotImplementedError("Timestep 4 frame VFI model")
    def non_timestep_inference(frame_0, frame_1, frame_2, frame_3, n):        
        middle = return_middle_frame_function(frame_0, frame_1, None, *return_middle_frame_function_args)
        if n == 1:
            return [middle]
        first_half = non_timestep_inference(frame_0, middle, n=n//2)
        second_half = non_timestep_inference(middle, frame_1, n=n//2)
        if n%2:
            return [*first_half, middle, *second_half]
        else:
            return [*first_half, *second_half] """