Add HunyuanVideo 1.5 image-to-video modular blocks

Author: akshan-main

src/diffusers/__init__.py

@@ -453,6 +453,7 @@
             "HeliosPyramidDistilledModularPipeline",
             "HeliosPyramidModularPipeline",
             "HunyuanVideo15Blocks",
+            "HunyuanVideo15Image2VideoBlocks",
             "HunyuanVideo15ModularPipeline",
             "QwenImageAutoBlocks",
             "QwenImageEditAutoBlocks",

src/diffusers/modular_pipelines/__init__.py

@@ -90,6 +90,7 @@
     ]
     _import_structure["hunyuan_video1_5"] = [
         "HunyuanVideo15Blocks",
+        "HunyuanVideo15Image2VideoBlocks",
         "HunyuanVideo15ModularPipeline",
     ]
     _import_structure["z_image"] = [
@@ -144,7 +145,7 @@
             QwenImageLayeredModularPipeline,
             QwenImageModularPipeline,
         )
-        from .hunyuan_video1_5 import HunyuanVideo15Blocks, HunyuanVideo15ModularPipeline
+        from .hunyuan_video1_5 import HunyuanVideo15Blocks, HunyuanVideo15Image2VideoBlocks, HunyuanVideo15ModularPipeline
         from .stable_diffusion_xl import StableDiffusionXLAutoBlocks, StableDiffusionXLModularPipeline
         from .wan import (
             Wan22Blocks,

src/diffusers/modular_pipelines/hunyuan_video1_5/__init__.py

@@ -21,7 +21,7 @@
 
     _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
 else:
-    _import_structure["modular_blocks_hunyuan_video1_5"] = ["HunyuanVideo15Blocks"]
+    _import_structure["modular_blocks_hunyuan_video1_5"] = ["HunyuanVideo15Blocks", "HunyuanVideo15Image2VideoBlocks"]
     _import_structure["modular_pipeline"] = ["HunyuanVideo15ModularPipeline"]
 
 if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
@@ -31,7 +31,7 @@
     except OptionalDependencyNotAvailable:
         from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
     else:
-        from .modular_blocks_hunyuan_video1_5 import HunyuanVideo15Blocks
+        from .modular_blocks_hunyuan_video1_5 import HunyuanVideo15Blocks, HunyuanVideo15Image2VideoBlocks
         from .modular_pipeline import HunyuanVideo15ModularPipeline
 else:
     import sys

src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py

@@ -222,3 +222,125 @@ def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineSta
 
         self.set_block_state(state, block_state)
         return components, state
+
+
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
+def retrieve_latents(encoder_output, generator=None, sample_mode="sample"):
+    if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
+        return encoder_output.latent_dist.sample(generator)
+    elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
+        return encoder_output.latent_dist.mode()
+    elif hasattr(encoder_output, "latents"):
+        return encoder_output.latents
+    raise AttributeError("Could not access latents of provided encoder_output")
+
+
+class HunyuanVideo15Image2VideoPrepareLatentsStep(ModularPipelineBlocks):
+    model_name = "hunyuan-video-1.5"
+
+    @property
+    def description(self) -> str:
+        return "Prepare latents, conditioning latents, mask, and image_embeds for I2V"
+
+    @property
+    def expected_components(self) -> list[ComponentSpec]:
+        from ...models import AutoencoderKLHunyuanVideo15
+        from ...pipelines.hunyuan_video1_5.image_processor import HunyuanVideo15ImageProcessor
+        from transformers import SiglipVisionModel, SiglipImageProcessor
+        return [
+            ComponentSpec("vae", AutoencoderKLHunyuanVideo15),
+            ComponentSpec(
+                "video_processor",
+                HunyuanVideo15ImageProcessor,
+                config=FrozenDict({"vae_scale_factor": 16}),
+                default_creation_method="from_config",
+            ),
+            ComponentSpec("image_encoder", SiglipVisionModel),
+            ComponentSpec("feature_extractor", SiglipImageProcessor),
+        ]
+
+    @property
+    def inputs(self) -> list[InputParam]:
+        return [
+            InputParam("image", required=True),
+            InputParam("num_frames", type_hint=int, default=121),
+            InputParam("latents", type_hint=torch.Tensor | None),
+            InputParam("num_videos_per_prompt", type_hint=int, default=1),
+            InputParam("generator"),
+            InputParam("batch_size", required=True, type_hint=int),
+            InputParam("dtype", type_hint=torch.dtype),
+        ]
+
+    @property
+    def intermediate_outputs(self) -> list[OutputParam]:
+        return [
+            OutputParam("latents", type_hint=torch.Tensor),
+            OutputParam("cond_latents_concat", type_hint=torch.Tensor),
+            OutputParam("mask_concat", type_hint=torch.Tensor),
+            OutputParam("image_embeds", type_hint=torch.Tensor),
+        ]
+
+    # Copied from pipeline_hunyuan_video1_5_image2video.py lines 756-839 with self->components
+    @torch.no_grad()
+    def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineState) -> PipelineState:
+        block_state = self.get_block_state(state)
+        device = components._execution_device
+        dtype = block_state.dtype
+
+        image = block_state.image
+        batch_size = block_state.batch_size * block_state.num_videos_per_prompt
+        num_frames = block_state.num_frames
+
+        # Resize/crop image to target resolution (line 756-759)
+        height, width = components.video_processor.calculate_default_height_width(
+            height=image.size[1], width=image.size[0], target_size=components.target_size
+        )
+        image = components.video_processor.resize(image, height=height, width=width, resize_mode="crop")
+
+        # Encode image with Siglip (lines 776-781)
+        image_encoder_dtype = next(components.image_encoder.parameters()).dtype
+        image_inputs = components.feature_extractor.preprocess(
+            images=image, do_resize=True, return_tensors="pt", do_convert_rgb=True
+        )
+        image_inputs = image_inputs.to(device=device, dtype=image_encoder_dtype)
+        image_embeds = components.image_encoder(**image_inputs).last_hidden_state
+        image_embeds = image_embeds.repeat(batch_size, 1, 1)
+        block_state.image_embeds = image_embeds.to(device=device, dtype=dtype)
+
+        # Prepare latents (lines 818-829)
+        latents = block_state.latents
+        if latents is not None:
+            latents = latents.to(device=device, dtype=dtype)
+        else:
+            shape = (
+                batch_size,
+                components.num_channels_latents,
+                (num_frames - 1) // components.vae_scale_factor_temporal + 1,
+                int(height) // components.vae_scale_factor_spatial,
+                int(width) // components.vae_scale_factor_spatial,
+            )
+            latents = randn_tensor(shape, generator=block_state.generator, device=device, dtype=dtype)
+        block_state.latents = latents
+
+        # Prepare cond latents and mask (lines 594-632, 831-839)
+        b, c, f, h, w = latents.shape
+
+        # Copied from _get_image_latents (lines 375-388) with self->components
+        vae_dtype = components.vae.dtype
+        image_tensor = components.video_processor.preprocess(
+            image, height=h * components.vae_scale_factor_spatial, width=w * components.vae_scale_factor_spatial
+        ).to(device, dtype=vae_dtype)
+        image_tensor = image_tensor.unsqueeze(2)
+        image_latents = retrieve_latents(components.vae.encode(image_tensor), sample_mode="argmax")
+        image_latents = image_latents * components.vae.config.scaling_factor
+
+        latent_condition = image_latents.repeat(batch_size, 1, f, 1, 1)
+        latent_condition[:, :, 1:, :, :] = 0
+        block_state.cond_latents_concat = latent_condition.to(device=device, dtype=dtype)
+
+        latent_mask = torch.zeros(b, 1, f, h, w, dtype=dtype, device=device)
+        latent_mask[:, :, 0, :, :] = 1.0
+        block_state.mask_concat = latent_mask
+
+        self.set_block_state(state, block_state)
+        return components, state

src/diffusers/modular_pipelines/hunyuan_video1_5/denoise.py

@@ -237,3 +237,122 @@ def description(self) -> str:
             " - `HunyuanVideo15LoopAfterDenoiser`\n"
             "This block supports text-to-video tasks."
         )
+
+
+class HunyuanVideo15Image2VideoLoopDenoiser(ModularPipelineBlocks):
+    model_name = "hunyuan-video-1.5"
+
+    def __init__(self, guider_input_fields=None):
+        if guider_input_fields is None:
+            guider_input_fields = {
+                "encoder_hidden_states": ("prompt_embeds", "negative_prompt_embeds"),
+                "encoder_attention_mask": ("prompt_embeds_mask", "negative_prompt_embeds_mask"),
+                "encoder_hidden_states_2": ("prompt_embeds_2", "negative_prompt_embeds_2"),
+                "encoder_attention_mask_2": ("prompt_embeds_mask_2", "negative_prompt_embeds_mask_2"),
+            }
+        if not isinstance(guider_input_fields, dict):
+            raise ValueError(f"guider_input_fields must be a dictionary but is {type(guider_input_fields)}")
+        self._guider_input_fields = guider_input_fields
+        super().__init__()
+
+    @property
+    def expected_components(self) -> list[ComponentSpec]:
+        return [
+            ComponentSpec(
+                "guider",
+                ClassifierFreeGuidance,
+                config=FrozenDict({"guidance_scale": 7.5}),
+                default_creation_method="from_config",
+            ),
+            ComponentSpec("transformer", HunyuanVideo15Transformer3DModel),
+        ]
+
+    @property
+    def description(self) -> str:
+        return "I2V denoiser with MeanFlow timestep_r support"
+
+    @property
+    def inputs(self) -> list[InputParam]:
+        inputs = [
+            InputParam("attention_kwargs"),
+            InputParam("num_inference_steps", required=True, type_hint=int),
+            InputParam("image_embeds", type_hint=torch.Tensor),
+            InputParam("timesteps", required=True, type_hint=torch.Tensor),
+        ]
+        for value in self._guider_input_fields.values():
+            if isinstance(value, tuple):
+                inputs.append(InputParam(name=value[0], required=True, type_hint=torch.Tensor))
+                for neg_name in value[1:]:
+                    inputs.append(InputParam(name=neg_name, type_hint=torch.Tensor))
+            else:
+                inputs.append(InputParam(name=value, required=True, type_hint=torch.Tensor))
+        return inputs
+
+    # Copied from pipeline_hunyuan_video1_5_image2video.py lines 853-912 with self->components
+    @torch.no_grad()
+    def __call__(
+        self, components: HunyuanVideo15ModularPipeline, block_state: BlockState, i: int, t: torch.Tensor
+    ) -> PipelineState:
+        timestep = t.expand(block_state.latent_model_input.shape[0]).to(block_state.latent_model_input.dtype)
+
+        # MeanFlow timestep_r (lines 855-862)
+        if components.transformer.config.use_meanflow:
+            if i == len(block_state.timesteps) - 1:
+                timestep_r = torch.tensor([0.0], device=timestep.device)
+            else:
+                timestep_r = block_state.timesteps[i + 1]
+            timestep_r = timestep_r.expand(block_state.latents.shape[0]).to(block_state.latents.dtype)
+        else:
+            timestep_r = None
+
+        guider_inputs = {
+            input_name: tuple(getattr(block_state, v) for v in value) if isinstance(value, tuple) else getattr(block_state, value)
+            for input_name, value in self._guider_input_fields.items()
+        }
+
+        components.guider.set_state(step=i, num_inference_steps=block_state.num_inference_steps, timestep=t)
+        guider_state = components.guider.prepare_inputs(guider_inputs)
+
+        for guider_state_batch in guider_state:
+            components.guider.prepare_models(components.transformer)
+
+            cond_kwargs = {
+                input_name: getattr(guider_state_batch, input_name) for input_name in guider_inputs.keys()
+            }
+
+            context_name = getattr(guider_state_batch, components.guider._identifier_key)
+            with components.transformer.cache_context(context_name):
+                guider_state_batch.noise_pred = components.transformer(
+                    hidden_states=block_state.latent_model_input,
+                    image_embeds=block_state.image_embeds,
+                    timestep=timestep,
+                    timestep_r=timestep_r,
+                    attention_kwargs=block_state.attention_kwargs,
+                    return_dict=False,
+                    **cond_kwargs,
+                )[0]
+
+            components.guider.cleanup_models(components.transformer)
+
+        block_state.noise_pred = components.guider(guider_state)[0]
+
+        return components, block_state
+
+
+class HunyuanVideo15Image2VideoDenoiseStep(HunyuanVideo15DenoiseLoopWrapper):
+    block_classes = [
+        HunyuanVideo15LoopBeforeDenoiser,
+        HunyuanVideo15Image2VideoLoopDenoiser(),
+        HunyuanVideo15LoopAfterDenoiser,
+    ]
+    block_names = ["before_denoiser", "denoiser", "after_denoiser"]
+
+    @property
+    def description(self) -> str:
+        return (
+            "Denoise step for image-to-video with MeanFlow support.\n"
+            "At each iteration:\n"
+            " - `HunyuanVideo15LoopBeforeDenoiser`\n"
+            " - `HunyuanVideo15Image2VideoLoopDenoiser`\n"
+            " - `HunyuanVideo15LoopAfterDenoiser`"
+        )

src/diffusers/modular_pipelines/hunyuan_video1_5/modular_blocks_hunyuan_video1_5.py

@@ -16,12 +16,13 @@
 from ..modular_pipeline import SequentialPipelineBlocks
 from ..modular_pipeline_utils import OutputParam
 from .before_denoise import (
+    HunyuanVideo15Image2VideoPrepareLatentsStep,
     HunyuanVideo15PrepareLatentsStep,
     HunyuanVideo15SetTimestepsStep,
     HunyuanVideo15TextInputStep,
 )
 from .decoders import HunyuanVideo15VaeDecoderStep
-from .denoise import HunyuanVideo15DenoiseStep
+from .denoise import HunyuanVideo15DenoiseStep, HunyuanVideo15Image2VideoDenoiseStep
 from .encoders import HunyuanVideo15TextEncoderStep
 
 
@@ -65,3 +66,42 @@ def description(self):
     @property
     def outputs(self):
         return [OutputParam.template("videos")]
+
+
+# auto_docstring
+class HunyuanVideo15Image2VideoCoreDenoiseStep(SequentialPipelineBlocks):
+    model_name = "hunyuan-video-1.5"
+    block_classes = [
+        HunyuanVideo15TextInputStep,
+        HunyuanVideo15SetTimestepsStep,
+        HunyuanVideo15Image2VideoPrepareLatentsStep,
+        HunyuanVideo15Image2VideoDenoiseStep,
+    ]
+    block_names = ["input", "set_timesteps", "prepare_latents", "denoise"]
+
+    @property
+    def description(self):
+        return "Denoise block for image-to-video that takes encoded conditions and runs the denoising process."
+
+    @property
+    def outputs(self):
+        return [OutputParam.template("latents")]
+
+
+# auto_docstring
+class HunyuanVideo15Image2VideoBlocks(SequentialPipelineBlocks):
+    model_name = "hunyuan-video-1.5"
+    block_classes = [
+        HunyuanVideo15TextEncoderStep,
+        HunyuanVideo15Image2VideoCoreDenoiseStep,
+        HunyuanVideo15VaeDecoderStep,
+    ]
+    block_names = ["text_encoder", "denoise", "decode"]
+
+    @property
+    def description(self):
+        return "Modular pipeline blocks for HunyuanVideo 1.5 image-to-video."
+
+    @property
+    def outputs(self):
+        return [OutputParam.template("videos")]