Inline all standard pipeline methods, remove runtime dependency

Author: akshan-main

src/diffusers/modular_pipelines/hunyuan_video1_5/before_denoise.py

@@ -18,7 +18,6 @@
 import torch
 
 from ...models import HunyuanVideo15Transformer3DModel
-from ...pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5 import HunyuanVideo15Pipeline
 from ...schedulers import FlowMatchEulerDiscreteScheduler
 from ...utils import logging
 from ...utils.torch_utils import randn_tensor
@@ -169,14 +168,13 @@ def intermediate_outputs(self) -> list[OutputParam]:
             OutputParam("image_embeds", type_hint=torch.Tensor),
         ]
 
-    # Copied from pipeline_hunyuan_video1_5.py lines 652-655, 706-725
+    # Copied from pipeline_hunyuan_video1_5.py lines 652-655, 477-524, 706-725 with self->components
     @torch.no_grad()
     def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineState) -> PipelineState:
         block_state = self.get_block_state(state)
         device = components._execution_device
         dtype = block_state.dtype
 
-        # Calculate default height/width if not provided (line 652-655)
         height = block_state.height
         width = block_state.width
         if height is None and width is None:
@@ -187,28 +185,33 @@ def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineSta
         batch_size = block_state.batch_size * block_state.num_videos_per_prompt
         num_frames = block_state.num_frames
 
-        # Copied from HunyuanVideo15Pipeline.prepare_latents (lines 477-505, 707-717)
-        block_state.latents = HunyuanVideo15Pipeline.prepare_latents(
-            components,
-            batch_size,
-            components.num_channels_latents,
-            height,
-            width,
-            num_frames,
-            dtype,
-            device,
-            block_state.generator,
-            block_state.latents,
-        )
+        # Copied from HunyuanVideo15Pipeline.prepare_latents with self->components
+        latents = block_state.latents
+        if latents is not None:
+            latents = latents.to(device=device, dtype=dtype)
+        else:
+            shape = (
+                batch_size,
+                components.num_channels_latents,
+                (num_frames - 1) // components.vae_scale_factor_temporal + 1,
+                int(height) // components.vae_scale_factor_spatial,
+                int(width) // components.vae_scale_factor_spatial,
+            )
+            if isinstance(block_state.generator, list) and len(block_state.generator) != batch_size:
+                raise ValueError(
+                    f"You have passed a list of generators of length {len(block_state.generator)}, but requested an effective batch"
+                    f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+                )
+            latents = randn_tensor(shape, generator=block_state.generator, device=device, dtype=dtype)
 
-        # Copied from HunyuanVideo15Pipeline.prepare_cond_latents_and_mask (lines 508-524, 718)
-        cond_latents_concat, mask_concat = HunyuanVideo15Pipeline.prepare_cond_latents_and_mask(
-            components, block_state.latents, dtype, device
-        )
-        block_state.cond_latents_concat = cond_latents_concat
-        block_state.mask_concat = mask_concat
+        block_state.latents = latents
+
+        # Copied from HunyuanVideo15Pipeline.prepare_cond_latents_and_mask with self->components
+        b, c, f, h, w = latents.shape
+        block_state.cond_latents_concat = torch.zeros(b, c, f, h, w, dtype=dtype, device=device)
+        block_state.mask_concat = torch.zeros(b, 1, f, h, w, dtype=dtype, device=device)
 
-        # T2V: zero image_embeds (line 719-725)
+        # T2V: zero image_embeds
         block_state.image_embeds = torch.zeros(
             block_state.batch_size,
             components.vision_num_semantic_tokens,

src/diffusers/modular_pipelines/hunyuan_video1_5/encoders.py

@@ -12,12 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
+
 import torch
 from transformers import ByT5Tokenizer, Qwen2_5_VLTextModel, Qwen2TokenizerFast, T5EncoderModel
 
 from ...configuration_utils import FrozenDict
 from ...guiders import ClassifierFreeGuidance
-from ...pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5 import HunyuanVideo15Pipeline
 from ...utils import logging
 from ..modular_pipeline import ModularPipelineBlocks, PipelineState
 from ..modular_pipeline_utils import ComponentSpec, InputParam, OutputParam
@@ -27,6 +28,111 @@
 logger = logging.get_logger(__name__)
 
 
+# Copied from diffusers.pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5.format_text_input
+def format_text_input(prompt, system_message):
+    return [
+        [{"role": "system", "content": system_message}, {"role": "user", "content": p if p else " "}] for p in prompt
+    ]
+
+
+# Copied from diffusers.pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5.extract_glyph_texts
+def extract_glyph_texts(prompt):
+    pattern = r"\"(.*?)\"|\"(.*?)\""
+    matches = re.findall(pattern, prompt)
+    result = [match[0] or match[1] for match in matches]
+    result = list(dict.fromkeys(result)) if len(result) > 1 else result
+    if result:
+        formatted_result = ". ".join([f'Text "{text}"' for text in result]) + ". "
+    else:
+        formatted_result = None
+    return formatted_result
+
+
+# Copied from diffusers.pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5.HunyuanVideo15Pipeline._get_mllm_prompt_embeds
+def _get_mllm_prompt_embeds(
+    text_encoder,
+    tokenizer,
+    prompt,
+    device,
+    tokenizer_max_length=1000,
+    num_hidden_layers_to_skip=2,
+    # fmt: off
+    system_message="You are a helpful assistant. Describe the video by detailing the following aspects: \
+    1. The main content and theme of the video. \
+    2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. \
+    3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. \
+    4. background environment, light, style and atmosphere. \
+    5. camera angles, movements, and transitions used in the video.",
+    # fmt: on
+    crop_start=108,
+):
+    prompt = [prompt] if isinstance(prompt, str) else prompt
+    prompt = format_text_input(prompt, system_message)
+
+    text_inputs = tokenizer.apply_chat_template(
+        prompt,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        padding="max_length",
+        max_length=tokenizer_max_length + crop_start,
+        truncation=True,
+        return_tensors="pt",
+    )
+
+    text_input_ids = text_inputs.input_ids.to(device=device)
+    prompt_attention_mask = text_inputs.attention_mask.to(device=device)
+
+    prompt_embeds = text_encoder(
+        input_ids=text_input_ids,
+        attention_mask=prompt_attention_mask,
+        output_hidden_states=True,
+    ).hidden_states[-(num_hidden_layers_to_skip + 1)]
+
+    if crop_start is not None and crop_start > 0:
+        prompt_embeds = prompt_embeds[:, crop_start:]
+        prompt_attention_mask = prompt_attention_mask[:, crop_start:]
+
+    return prompt_embeds, prompt_attention_mask
+
+
+# Copied from diffusers.pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5.HunyuanVideo15Pipeline._get_byt5_prompt_embeds
+def _get_byt5_prompt_embeds(tokenizer, text_encoder, prompt, device, tokenizer_max_length=256):
+    prompt = [prompt] if isinstance(prompt, str) else prompt
+    glyph_texts = [extract_glyph_texts(p) for p in prompt]
+
+    prompt_embeds_list = []
+    prompt_embeds_mask_list = []
+
+    for glyph_text in glyph_texts:
+        if glyph_text is None:
+            glyph_text_embeds = torch.zeros(
+                (1, tokenizer_max_length, text_encoder.config.d_model), device=device, dtype=text_encoder.dtype
+            )
+            glyph_text_embeds_mask = torch.zeros((1, tokenizer_max_length), device=device, dtype=torch.int64)
+        else:
+            txt_tokens = tokenizer(
+                glyph_text,
+                padding="max_length",
+                max_length=tokenizer_max_length,
+                truncation=True,
+                add_special_tokens=True,
+                return_tensors="pt",
+            ).to(device)
+
+            glyph_text_embeds = text_encoder(
+                input_ids=txt_tokens.input_ids,
+                attention_mask=txt_tokens.attention_mask.float(),
+            )[0]
+            glyph_text_embeds = glyph_text_embeds.to(device=device)
+            glyph_text_embeds_mask = txt_tokens.attention_mask.to(device=device)
+
+        prompt_embeds_list.append(glyph_text_embeds)
+        prompt_embeds_mask_list.append(glyph_text_embeds_mask)
+
+    return torch.cat(prompt_embeds_list, dim=0), torch.cat(prompt_embeds_mask_list, dim=0)
+
+
 class HunyuanVideo15TextEncoderStep(ModularPipelineBlocks):
     model_name = "hunyuan-video-1.5"
 
@@ -78,38 +184,29 @@ def intermediate_outputs(self) -> list[OutputParam]:
             OutputParam("negative_prompt_embeds_mask_2", type_hint=torch.Tensor, kwargs_type="denoiser_input_fields"),
         ]
 
-    # Copied from HunyuanVideo15Pipeline.encode_prompt
-    @torch.no_grad()
-    def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineState) -> PipelineState:
-        block_state = self.get_block_state(state)
-        device = components._execution_device
-        dtype = components.transformer.dtype
-
-        prompt = block_state.prompt
-        negative_prompt = block_state.negative_prompt
-        num_videos_per_prompt = block_state.num_videos_per_prompt
-
-        if prompt is not None and isinstance(prompt, str):
-            batch_size = 1
-        elif prompt is not None and isinstance(prompt, list):
-            batch_size = len(prompt)
-        elif getattr(block_state, "prompt_embeds", None) is not None:
-            batch_size = block_state.prompt_embeds.shape[0]
-        else:
-            batch_size = 1
-
-        # Encode positive prompt - copied from HunyuanVideo15Pipeline.encode_prompt
-        prompt_embeds = getattr(block_state, "prompt_embeds", None)
-        prompt_embeds_mask = getattr(block_state, "prompt_embeds_mask", None)
-        prompt_embeds_2 = getattr(block_state, "prompt_embeds_2", None)
-        prompt_embeds_mask_2 = getattr(block_state, "prompt_embeds_mask_2", None)
+    # Copied from diffusers.pipelines.hunyuan_video1_5.pipeline_hunyuan_video1_5.HunyuanVideo15Pipeline.encode_prompt with self->components
+    @staticmethod
+    def encode_prompt(
+        components,
+        prompt,
+        device=None,
+        dtype=None,
+        batch_size=1,
+        num_videos_per_prompt=1,
+        prompt_embeds=None,
+        prompt_embeds_mask=None,
+        prompt_embeds_2=None,
+        prompt_embeds_mask_2=None,
+    ):
+        device = device or components._execution_device
+        dtype = dtype or components.text_encoder.dtype
 
         if prompt is None:
             prompt = [""] * batch_size
         prompt = [prompt] if isinstance(prompt, str) else prompt
 
         if prompt_embeds is None:
-            prompt_embeds, prompt_embeds_mask = HunyuanVideo15Pipeline._get_mllm_prompt_embeds(
+            prompt_embeds, prompt_embeds_mask = _get_mllm_prompt_embeds(
                 tokenizer=components.tokenizer,
                 text_encoder=components.text_encoder,
                 prompt=prompt,
@@ -120,7 +217,7 @@ def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineSta
             )
 
         if prompt_embeds_2 is None:
-            prompt_embeds_2, prompt_embeds_mask_2 = HunyuanVideo15Pipeline._get_byt5_prompt_embeds(
+            prompt_embeds_2, prompt_embeds_mask_2 = _get_byt5_prompt_embeds(
                 tokenizer=components.tokenizer_2,
                 text_encoder=components.text_encoder_2,
                 prompt=prompt,
@@ -136,57 +233,69 @@ def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineSta
         prompt_embeds_2 = prompt_embeds_2.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len_2, -1)
         prompt_embeds_mask_2 = prompt_embeds_mask_2.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len_2)
 
-        block_state.prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
-        block_state.prompt_embeds_mask = prompt_embeds_mask.to(dtype=dtype, device=device)
-        block_state.prompt_embeds_2 = prompt_embeds_2.to(dtype=dtype, device=device)
-        block_state.prompt_embeds_mask_2 = prompt_embeds_mask_2.to(dtype=dtype, device=device)
+        prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
+        prompt_embeds_mask = prompt_embeds_mask.to(dtype=dtype, device=device)
+        prompt_embeds_2 = prompt_embeds_2.to(dtype=dtype, device=device)
+        prompt_embeds_mask_2 = prompt_embeds_mask_2.to(dtype=dtype, device=device)
+
+        return prompt_embeds, prompt_embeds_mask, prompt_embeds_2, prompt_embeds_mask_2
+
+    @torch.no_grad()
+    def __call__(self, components: HunyuanVideo15ModularPipeline, state: PipelineState) -> PipelineState:
+        block_state = self.get_block_state(state)
+        device = components._execution_device
+        dtype = components.transformer.dtype
+
+        prompt = block_state.prompt
+        negative_prompt = block_state.negative_prompt
+        num_videos_per_prompt = block_state.num_videos_per_prompt
+
+        if prompt is not None and isinstance(prompt, str):
+            batch_size = 1
+        elif prompt is not None and isinstance(prompt, list):
+            batch_size = len(prompt)
+        elif getattr(block_state, "prompt_embeds", None) is not None:
+            batch_size = block_state.prompt_embeds.shape[0]
+        else:
+            batch_size = 1
+
+        (
+            block_state.prompt_embeds,
+            block_state.prompt_embeds_mask,
+            block_state.prompt_embeds_2,
+            block_state.prompt_embeds_mask_2,
+        ) = self.encode_prompt(
+            components,
+            prompt=prompt,
+            device=device,
+            dtype=dtype,
+            batch_size=batch_size,
+            num_videos_per_prompt=num_videos_per_prompt,
+            prompt_embeds=getattr(block_state, "prompt_embeds", None),
+            prompt_embeds_mask=getattr(block_state, "prompt_embeds_mask", None),
+            prompt_embeds_2=getattr(block_state, "prompt_embeds_2", None),
+            prompt_embeds_mask_2=getattr(block_state, "prompt_embeds_mask_2", None),
+        )
 
-        # Encode negative prompt if guider needs it
         if components.requires_unconditional_embeds:
-            neg_prompt_embeds = getattr(block_state, "negative_prompt_embeds", None)
-            neg_prompt_embeds_mask = getattr(block_state, "negative_prompt_embeds_mask", None)
-            neg_prompt_embeds_2 = getattr(block_state, "negative_prompt_embeds_2", None)
-            neg_prompt_embeds_mask_2 = getattr(block_state, "negative_prompt_embeds_mask_2", None)
-
-            neg_prompt = negative_prompt
-            if neg_prompt is None:
-                neg_prompt = [""] * batch_size
-            neg_prompt = [neg_prompt] if isinstance(neg_prompt, str) else neg_prompt
-
-            if neg_prompt_embeds is None:
-                neg_prompt_embeds, neg_prompt_embeds_mask = HunyuanVideo15Pipeline._get_mllm_prompt_embeds(
-                    tokenizer=components.tokenizer,
-                    text_encoder=components.text_encoder,
-                    prompt=neg_prompt,
-                    device=device,
-                    tokenizer_max_length=components.tokenizer_max_length,
-                    system_message=components.system_message,
-                    crop_start=components.prompt_template_encode_start_idx,
-                )
-
-            if neg_prompt_embeds_2 is None:
-                neg_prompt_embeds_2, neg_prompt_embeds_mask_2 = HunyuanVideo15Pipeline._get_byt5_prompt_embeds(
-                    tokenizer=components.tokenizer_2,
-                    text_encoder=components.text_encoder_2,
-                    prompt=neg_prompt,
-                    device=device,
-                    tokenizer_max_length=components.tokenizer_2_max_length,
-                )
-
-            _, seq_len, _ = neg_prompt_embeds.shape
-            neg_prompt_embeds = neg_prompt_embeds.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len, -1)
-            neg_prompt_embeds_mask = neg_prompt_embeds_mask.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len)
-
-            _, seq_len_2, _ = neg_prompt_embeds_2.shape
-            neg_prompt_embeds_2 = neg_prompt_embeds_2.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len_2, -1)
-            neg_prompt_embeds_mask_2 = neg_prompt_embeds_mask_2.repeat(1, num_videos_per_prompt, 1).view(batch_size * num_videos_per_prompt, seq_len_2)
-
-            block_state.negative_prompt_embeds = neg_prompt_embeds.to(dtype=dtype, device=device)
-            block_state.negative_prompt_embeds_mask = neg_prompt_embeds_mask.to(dtype=dtype, device=device)
-            block_state.negative_prompt_embeds_2 = neg_prompt_embeds_2.to(dtype=dtype, device=device)
-            block_state.negative_prompt_embeds_mask_2 = neg_prompt_embeds_mask_2.to(dtype=dtype, device=device)
-
-        # Pass batch_size downstream
+            (
+                block_state.negative_prompt_embeds,
+                block_state.negative_prompt_embeds_mask,
+                block_state.negative_prompt_embeds_2,
+                block_state.negative_prompt_embeds_mask_2,
+            ) = self.encode_prompt(
+                components,
+                prompt=negative_prompt,
+                device=device,
+                dtype=dtype,
+                batch_size=batch_size,
+                num_videos_per_prompt=num_videos_per_prompt,
+                prompt_embeds=getattr(block_state, "negative_prompt_embeds", None),
+                prompt_embeds_mask=getattr(block_state, "negative_prompt_embeds_mask", None),
+                prompt_embeds_2=getattr(block_state, "negative_prompt_embeds_2", None),
+                prompt_embeds_mask_2=getattr(block_state, "negative_prompt_embeds_mask_2", None),
+            )
+
         state.set("batch_size", batch_size)
 
         self.set_block_state(state, block_state)