|
| 1 | +import unittest |
| 2 | + |
| 3 | +import numpy as np |
| 4 | +import torch |
| 5 | +from PIL import Image |
| 6 | +from transformers import Qwen2TokenizerFast, Qwen3Config, Qwen3ForCausalLM |
| 7 | + |
| 8 | +from diffusers import ( |
| 9 | + AutoencoderKLFlux2, |
| 10 | + FlowMatchEulerDiscreteScheduler, |
| 11 | + Flux2KleinKVPipeline, |
| 12 | + Flux2Transformer2DModel, |
| 13 | +) |
| 14 | + |
| 15 | +from ...testing_utils import torch_device |
| 16 | +from ..test_pipelines_common import PipelineTesterMixin, check_qkv_fused_layers_exist |
| 17 | + |
| 18 | + |
| 19 | +class Flux2KleinKVPipelineFastTests(PipelineTesterMixin, unittest.TestCase): |
| 20 | + pipeline_class = Flux2KleinKVPipeline |
| 21 | + params = frozenset(["prompt", "height", "width", "prompt_embeds", "image"]) |
| 22 | + batch_params = frozenset(["prompt"]) |
| 23 | + |
| 24 | + test_xformers_attention = False |
| 25 | + test_layerwise_casting = True |
| 26 | + test_group_offloading = True |
| 27 | + |
| 28 | + supports_dduf = False |
| 29 | + |
| 30 | + def get_dummy_components(self, num_layers: int = 1, num_single_layers: int = 1): |
| 31 | + torch.manual_seed(0) |
| 32 | + transformer = Flux2Transformer2DModel( |
| 33 | + patch_size=1, |
| 34 | + in_channels=4, |
| 35 | + num_layers=num_layers, |
| 36 | + num_single_layers=num_single_layers, |
| 37 | + attention_head_dim=16, |
| 38 | + num_attention_heads=2, |
| 39 | + joint_attention_dim=16, |
| 40 | + timestep_guidance_channels=256, |
| 41 | + axes_dims_rope=[4, 4, 4, 4], |
| 42 | + guidance_embeds=False, |
| 43 | + ) |
| 44 | + |
| 45 | + # Create minimal Qwen3 config |
| 46 | + config = Qwen3Config( |
| 47 | + intermediate_size=16, |
| 48 | + hidden_size=16, |
| 49 | + num_hidden_layers=2, |
| 50 | + num_attention_heads=2, |
| 51 | + num_key_value_heads=2, |
| 52 | + vocab_size=151936, |
| 53 | + max_position_embeddings=512, |
| 54 | + ) |
| 55 | + torch.manual_seed(0) |
| 56 | + text_encoder = Qwen3ForCausalLM(config) |
| 57 | + |
| 58 | + # Use a simple tokenizer for testing |
| 59 | + tokenizer = Qwen2TokenizerFast.from_pretrained( |
| 60 | + "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration" |
| 61 | + ) |
| 62 | + |
| 63 | + torch.manual_seed(0) |
| 64 | + vae = AutoencoderKLFlux2( |
| 65 | + sample_size=32, |
| 66 | + in_channels=3, |
| 67 | + out_channels=3, |
| 68 | + down_block_types=("DownEncoderBlock2D",), |
| 69 | + up_block_types=("UpDecoderBlock2D",), |
| 70 | + block_out_channels=(4,), |
| 71 | + layers_per_block=1, |
| 72 | + latent_channels=1, |
| 73 | + norm_num_groups=1, |
| 74 | + use_quant_conv=False, |
| 75 | + use_post_quant_conv=False, |
| 76 | + ) |
| 77 | + |
| 78 | + scheduler = FlowMatchEulerDiscreteScheduler() |
| 79 | + |
| 80 | + return { |
| 81 | + "scheduler": scheduler, |
| 82 | + "text_encoder": text_encoder, |
| 83 | + "tokenizer": tokenizer, |
| 84 | + "transformer": transformer, |
| 85 | + "vae": vae, |
| 86 | + } |
| 87 | + |
| 88 | + def get_dummy_inputs(self, device, seed=0): |
| 89 | + if str(device).startswith("mps"): |
| 90 | + generator = torch.manual_seed(seed) |
| 91 | + else: |
| 92 | + generator = torch.Generator(device="cpu").manual_seed(seed) |
| 93 | + |
| 94 | + inputs = { |
| 95 | + "prompt": "a dog is dancing", |
| 96 | + "image": Image.new("RGB", (64, 64)), |
| 97 | + "generator": generator, |
| 98 | + "num_inference_steps": 2, |
| 99 | + "height": 8, |
| 100 | + "width": 8, |
| 101 | + "max_sequence_length": 64, |
| 102 | + "output_type": "np", |
| 103 | + "text_encoder_out_layers": (1,), |
| 104 | + } |
| 105 | + return inputs |
| 106 | + |
| 107 | + def test_fused_qkv_projections(self): |
| 108 | + device = "cpu" # ensure determinism for the device-dependent torch.Generator |
| 109 | + components = self.get_dummy_components() |
| 110 | + pipe = self.pipeline_class(**components) |
| 111 | + pipe = pipe.to(device) |
| 112 | + pipe.set_progress_bar_config(disable=None) |
| 113 | + |
| 114 | + inputs = self.get_dummy_inputs(device) |
| 115 | + image = pipe(**inputs).images |
| 116 | + original_image_slice = image[0, -3:, -3:, -1] |
| 117 | + |
| 118 | + pipe.transformer.fuse_qkv_projections() |
| 119 | + self.assertTrue( |
| 120 | + check_qkv_fused_layers_exist(pipe.transformer, ["to_qkv"]), |
| 121 | + ("Something wrong with the fused attention layers. Expected all the attention projections to be fused."), |
| 122 | + ) |
| 123 | + |
| 124 | + inputs = self.get_dummy_inputs(device) |
| 125 | + image = pipe(**inputs).images |
| 126 | + image_slice_fused = image[0, -3:, -3:, -1] |
| 127 | + |
| 128 | + pipe.transformer.unfuse_qkv_projections() |
| 129 | + inputs = self.get_dummy_inputs(device) |
| 130 | + image = pipe(**inputs).images |
| 131 | + image_slice_disabled = image[0, -3:, -3:, -1] |
| 132 | + |
| 133 | + self.assertTrue( |
| 134 | + np.allclose(original_image_slice, image_slice_fused, atol=1e-3, rtol=1e-3), |
| 135 | + ("Fusion of QKV projections shouldn't affect the outputs."), |
| 136 | + ) |
| 137 | + self.assertTrue( |
| 138 | + np.allclose(image_slice_fused, image_slice_disabled, atol=1e-3, rtol=1e-3), |
| 139 | + ("Outputs, with QKV projection fusion enabled, shouldn't change when fused QKV projections are disabled."), |
| 140 | + ) |
| 141 | + self.assertTrue( |
| 142 | + np.allclose(original_image_slice, image_slice_disabled, atol=1e-2, rtol=1e-2), |
| 143 | + ("Original outputs should match when fused QKV projections are disabled."), |
| 144 | + ) |
| 145 | + |
| 146 | + def test_image_output_shape(self): |
| 147 | + pipe = self.pipeline_class(**self.get_dummy_components()).to(torch_device) |
| 148 | + inputs = self.get_dummy_inputs(torch_device) |
| 149 | + |
| 150 | + height_width_pairs = [(32, 32), (72, 57)] |
| 151 | + for height, width in height_width_pairs: |
| 152 | + expected_height = height - height % (pipe.vae_scale_factor * 2) |
| 153 | + expected_width = width - width % (pipe.vae_scale_factor * 2) |
| 154 | + |
| 155 | + inputs.update({"height": height, "width": width}) |
| 156 | + image = pipe(**inputs).images[0] |
| 157 | + output_height, output_width, _ = image.shape |
| 158 | + self.assertEqual( |
| 159 | + (output_height, output_width), |
| 160 | + (expected_height, expected_width), |
| 161 | + f"Output shape {image.shape} does not match expected shape {(expected_height, expected_width)}", |
| 162 | + ) |
| 163 | + |
| 164 | + def test_without_image(self): |
| 165 | + device = "cpu" |
| 166 | + pipe = self.pipeline_class(**self.get_dummy_components()).to(device) |
| 167 | + inputs = self.get_dummy_inputs(device) |
| 168 | + del inputs["image"] |
| 169 | + image = pipe(**inputs).images |
| 170 | + self.assertEqual(image.shape, (1, 8, 8, 3)) |
| 171 | + |
| 172 | + @unittest.skip("Needs to be revisited") |
| 173 | + def test_encode_prompt_works_in_isolation(self): |
| 174 | + pass |
0 commit comments