From 02cb124a79fe103409548301817b8c37e11c0fab Mon Sep 17 00:00:00 2001 From: Mohamed-Ashraf273 Date: Fri, 26 Jun 2026 15:42:41 +0300 Subject: [PATCH 1/2] [OpenVINO] Support Youtu-VL-4B-Instruct with task image-text-to-text --- docs/source/openvino/models.mdx | 1 + optimum/exporters/openvino/model_configs.py | 203 +++++++++++++++ optimum/exporters/openvino/model_patcher.py | 123 +++++++++ optimum/exporters/openvino/utils.py | 1 + .../openvino/modeling_visual_language.py | 238 ++++++++++++++++++ tests/openvino/test_exporters_cli.py | 12 + tests/openvino/test_quantization.py | 1 + tests/openvino/test_seq2seq.py | 8 + tests/openvino/utils_tests.py | 8 + 9 files changed, 595 insertions(+) diff --git a/docs/source/openvino/models.mdx b/docs/source/openvino/models.mdx index 53b8cab389..f1a32e325b 100644 --- a/docs/source/openvino/models.mdx +++ b/docs/source/openvino/models.mdx @@ -173,6 +173,7 @@ Here is the list of the supported architectures : - XLM - XLM-RoBERTa - XVERSE +- Youtu_vl - Zamba2 ## [Diffusers](https://huggingface.co/docs/diffusers/index) diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index c2c937b80b..936a1e3553 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -171,6 +171,8 @@ SpeechT5ModelPatcher, VideoChatFlashQwenVisionEmbeddingModelPatcher, XverseModelPatcher, + YoutuVLLanguageModelPatcher, + YoutuVLVisionEmbMergerPatcher, Zamba2ModelPatcher, _get_model_attribute, ) @@ -324,6 +326,14 @@ def init_model_configs(): "transformers", "AutoModel", ) + TasksManager._CUSTOM_CLASSES[("pt", "llama4", "image-text-to-text")] = ( + "transformers", + "AutoModelForImageTextToText", + ) + TasksManager._CUSTOM_CLASSES[("pt", "youtu_vl", "image-text-to-text")] = ( + "transformers", + "AutoModelForCausalLM", + ) if is_diffusers_available() and "fill" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS: TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS["fill"] = "FluxFillPipeline" @@ -3802,6 +3812,199 @@ def __init__( self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config) +@register_in_tasks_manager( + "youtu_vl", *["text-generation", "text-generation-with-past"], library_name="transformers" +) +class YoutuVLTextOpenVINOConfig(MiniCPM3OpenVINOConfig): + # The Youtu language model is a DeepSeek-V2/V3 style decoder that uses multi-head + # latent attention (MLA). The key/value cache layout therefore matches MiniCPM3 / + # DeepSeek (k_head_dim = qk_nope_head_dim + qk_rope_head_dim, v_head_dim). The model + # is shipped as remote code, so unlike the in-library deepseek/minicpm3 configs we do + # not constrain the maximum transformers version. + MIN_TRANSFORMERS_VERSION = "4.49.0" + MAX_TRANSFORMERS_VERSION = "999.9.9" + _MODEL_PATCHER = YoutuVLLanguageModelPatcher + + +class YoutuVLVisionEmbeddingsDummyInputGenerator(DummyVisionInputGenerator): + # The Youtu-VL vision tower follows the Qwen2.5-VL windowed-attention design, so the + # vision part is split into a patch-embedding model and a transformer/merger model. + # Data-dependent operations (window indices, cu_seqlens based masks) are computed at + # runtime and fed into the merger as explicit inputs. + SUPPORTED_INPUT_NAMES = ( + "pixel_values", + "hidden_states", + "attention_mask", + "window_attention_mask", + "window_index", + "rotary_pos_emb", + ) + + def __init__( + self, + task: str, + normalized_config: NormalizedVisionConfig, + batch_size: int = 1, + **kwargs, + ): + self.task = task + self.normalized_config = normalized_config + vision_config = normalized_config.config + self.batch_size = 1 + self.patch_size = vision_config.patch_size + self.num_channels = vision_config.num_channels + self.hidden_size = vision_config.hidden_size + self.num_heads = vision_config.num_attention_heads + self.spatial_merge_size = 2 + self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size + # square grid of patches used for tracing; the corresponding axes are dynamic + self.spatial_dim = 8 + self.num_patches = self.spatial_dim * self.spatial_dim + # flattened patch dimension (num_channels * patch_size * patch_size) + self.embed_dim = self.num_channels * self.patch_size * self.patch_size + # rotary embedding dimension used by the vision transformer + self.rotary_dim = self.hidden_size // self.num_heads // 2 + + def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): + if input_name == "pixel_values": + shape = [self.batch_size, self.num_patches, self.embed_dim] + return self.random_float_tensor(shape, framework=framework, dtype=float_dtype) + if input_name == "hidden_states": + shape = [self.num_patches, self.hidden_size] + return self.random_float_tensor(shape, framework=framework, dtype=float_dtype) + if input_name in ["attention_mask", "window_attention_mask"]: + return self.random_mask_tensor( + [1, self.num_patches, self.num_patches], framework=framework, dtype=float_dtype + ) + if input_name == "rotary_pos_emb": + shape = [self.num_patches, self.rotary_dim] + return self.random_float_tensor(shape, framework=framework, dtype=float_dtype) + if input_name == "window_index": + length = self.num_patches // self.spatial_merge_unit + return self.random_int_tensor([length], max_value=length, framework=framework, dtype=int_dtype) + raise ValueError(f"Unsupported input name {input_name}") + + +@register_in_tasks_manager("youtu_vl", *["image-text-to-text"], library_name="transformers") +class YoutuVLOpenVINOConfig(BaseVLMOpenVINOConfig): + SUPPORTED_BEHAVIORS = [model_type.value for model_type in Qwen2VLConfigBehavior] + NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig + DUMMY_INPUT_GENERATOR_CLASSES = (YoutuVLVisionEmbeddingsDummyInputGenerator,) + MIN_TRANSFORMERS_VERSION = "4.49.0" + + def __init__( + self, + config: "PretrainedConfig", + task: str = "feature-extraction", + int_dtype: str = "int64", + float_dtype: str = "fp32", + behavior: Qwen2VLConfigBehavior = Qwen2VLConfigBehavior.VISION_EMBEDDINGS, + preprocessors: Optional[List[Any]] = None, + **kwargs, + ): + super().__init__( + config=config, + task=task, + int_dtype=int_dtype, + float_dtype=float_dtype, + preprocessors=preprocessors, + ) + self._behavior = behavior + self._orig_config = config + if ( + self._behavior in [Qwen2VLConfigBehavior.VISION_EMBEDDINGS, Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER] + and hasattr(config, "vision_config") + ): + self._config = config.vision_config + self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config) + + @staticmethod + def get_model_for_behavior(model, behavior: Union[str, Qwen2VLConfigBehavior]): + if isinstance(behavior, str) and not isinstance(behavior, Qwen2VLConfigBehavior): + behavior = Qwen2VLConfigBehavior(behavior) + + if behavior == Qwen2VLConfigBehavior.LANGUAGE: + return model + + if behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + # patch embedding (Siglip2VisionEmbeddingsWoPos) + vision_embeddings = model.siglip2.vision_model.embeddings + vision_embeddings.config = model.config.vision_config + return vision_embeddings + + if behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + # the encoder + post layernorm + merger are exported together; they are + # spread across several submodules of the top-level model, so we export the + # whole model and rely on the patcher to wire the right forward. + model.config.vision_config.fullatt_block_indexes = model.config.vision_config.fullatt_block_indexes + return model + + if behavior == Qwen2VLConfigBehavior.TEXT_EMBEDDINGS: + text_embedding = model.get_input_embeddings() + text_embedding.config = model.config + return text_embedding + + def with_behavior( + self, + behavior: Union[str, Qwen2VLConfigBehavior], + ): + if isinstance(behavior, str) and not isinstance(behavior, Qwen2VLConfigBehavior): + behavior = Qwen2VLConfigBehavior(behavior) + + if behavior == Qwen2VLConfigBehavior.TEXT_EMBEDDINGS: + return get_vlm_text_embeddings_config("youtu_vl", self._orig_config, self.int_dtype, self.float_dtype) + + if behavior == Qwen2VLConfigBehavior.LANGUAGE: + return get_vlm_text_generation_config( + "youtu_vl", + self._orig_config, + self.int_dtype, + self.float_dtype, + model_patcher=YoutuVLLanguageModelPatcher, + ) + + if behavior in [Qwen2VLConfigBehavior.VISION_EMBEDDINGS, Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER]: + return self.__class__( + self._orig_config, + task=self.task, + int_dtype=self.int_dtype, + float_dtype=self.float_dtype, + behavior=behavior, + preprocessors=self._preprocessors, + ) + + def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[Dict[str, Any]] = None): + model_kwargs = model_kwargs or {} + if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + return YoutuVLVisionEmbMergerPatcher(self, model, model_kwargs) + if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + return ModelPatcher(self, model, model_kwargs=model_kwargs) + return super().patch_model_for_export(model, model_kwargs) + + @property + def inputs(self) -> Dict[str, Dict[int, str]]: + if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + return {"pixel_values": {0: "batch_size", 1: "num_patches"}} + if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + return { + "hidden_states": {0: "sequence_length"}, + "attention_mask": {1: "sequence_length", 2: "sequence_length"}, + "window_attention_mask": {1: "sequence_length", 2: "sequence_length"}, + "window_index": {0: "unit_sequence_length"}, + "rotary_pos_emb": {0: "sequence_length"}, + } + return {} + + @property + def outputs(self) -> Dict[str, Dict[int, str]]: + if self._behavior in [ + Qwen2VLConfigBehavior.VISION_EMBEDDINGS, + Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER, + ]: + return {"last_hidden_state": {0: "seq_len"}} + return {} + + @register_in_tasks_manager("gemma3", *["image-text-to-text"], library_name="transformers") class Gemma3OpenVINOConfig(BaseVLMOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.50.0" diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index ed51e3d2f6..ac8626a936 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -10317,3 +10317,126 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) self._model.forward = self._model._orig_forward + +def _youtu_vl_vision_rotate_half(x): + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + +def _youtu_vl_apply_rotary_pos_emb_vision(q, k, cos, sin): + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2).float(), sin.unsqueeze(-2).float() + q_embed = (q * cos) + (_youtu_vl_vision_rotate_half(q) * sin) + k_embed = (k * cos) + (_youtu_vl_vision_rotate_half(k) * sin) + return q_embed.to(orig_q_dtype), k_embed.to(orig_k_dtype) + +def _youtu_vl_vision_sdpa_attn_forward(self, hidden_states, attention_mask, position_embeddings): + seq_length = hidden_states.shape[0] + q = self.q_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + k = self.k_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + v = self.v_proj(hidden_states).view(seq_length, self.num_heads, self.head_dim) + cos, sin = position_embeddings + q, k = _youtu_vl_apply_rotary_pos_emb_vision(q, k, cos, sin) + q = q.transpose(0, 1).unsqueeze(0) + k = k.transpose(0, 1).unsqueeze(0) + v = v.transpose(0, 1).unsqueeze(0) + attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attention_mask) + attn_output = attn_output.squeeze(0).transpose(0, 1).reshape(seq_length, -1).to(hidden_states.dtype) + return self.out_proj(attn_output), None + +def _youtu_vl_vision_block_forward(self, hidden_states, attention_mask, position_embeddings): + residual = hidden_states + hidden_states = self.layer_norm1(hidden_states) + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, attention_mask=attention_mask, position_embeddings=position_embeddings + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return (hidden_states,) + +class YoutuVLVisionEmbMergerPatcher(ModelPatcher): + def __init__( + self, + config: "OnnxConfig", + model: "PreTrainedModel", + model_kwargs: Dict[str, Any] = None, + ): + model.__orig_forward = model.forward + + # Modified from the Youtu-VL Siglip2 windowed vision encoder. The data-dependent + # window indices / cu_seqlens-based attention masks are computed at runtime and fed + # as explicit inputs (attention_mask, window_attention_mask, window_index, + # rotary_pos_emb), mirroring the Qwen2.5-VL OpenVINO export. + def image_embed_forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + window_attention_mask: torch.Tensor, + window_index: torch.Tensor, + rotary_pos_emb: torch.Tensor, + ) -> torch.Tensor: + encoder = self.siglip2.vision_model.encoder + post_layernorm = self.siglip2.vision_model.post_layernorm + spatial_merge_unit = encoder.spatial_merge_unit + fullatt_block_indexes = self.config.vision_config.fullatt_block_indexes + num_layers = len(encoder.layers) + + seq_len = hidden_states.shape[0] + hidden_states = hidden_states.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1) + hidden_states = hidden_states[window_index, :, :] + hidden_states = hidden_states.reshape(seq_len, -1) + rotary_pos_emb = rotary_pos_emb.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1) + rotary_pos_emb = rotary_pos_emb[window_index, :, :] + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1) + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) + position_embeddings = (emb.cos(), emb.sin()) + + for layer_num, blk in enumerate(encoder.layers): + if (1 + layer_num) % 8 == 0 or layer_num == num_layers - 1 or layer_num in fullatt_block_indexes: + attention_mask_now = attention_mask + else: + attention_mask_now = window_attention_mask + hidden_states = blk( + hidden_states, attention_mask=attention_mask_now, position_embeddings=position_embeddings + )[0] + + # restore the original (non-windowed) ordering before the patch merger + hidden_states = hidden_states.reshape(seq_len // spatial_merge_unit, spatial_merge_unit, -1) + reverse_indices = torch.argsort(window_index) + hidden_states = hidden_states[reverse_indices, :, :] + hidden_states = hidden_states.reshape(seq_len, -1) + + hidden_states = post_layernorm(hidden_states) + hidden_states = self.merger(hidden_states, None) + return hidden_states + + model.forward = types.MethodType(image_embed_forward, model) + super().__init__(config, model, model_kwargs) + + def __enter__(self): + super().__enter__() + for block in self._model.siglip2.vision_model.encoder.layers: + block._orig_forward = block.forward + block.forward = types.MethodType(_youtu_vl_vision_block_forward, block) + block.self_attn._orig_forward = block.self_attn.forward + block.self_attn.forward = types.MethodType(_youtu_vl_vision_sdpa_attn_forward, block.self_attn) + + def __exit__(self, exc_type, exc_value, traceback): + super().__exit__(exc_type, exc_value, traceback) + self._model.forward = self._model.__orig_forward + for block in self._model.siglip2.vision_model.encoder.layers: + block.forward = block._orig_forward + block.self_attn.forward = block.self_attn._orig_forward + +class YoutuVLLanguageModelPatcher(OVDecoderModelPatcher): + # The Youtu language model uses DeepSeek-style multi-head latent attention (MLA) + # with rotary embeddings computed inside the base model and SDPA in the attention + # block. These patterns are already torchscript/OpenVINO friendly, so no extra + # attention patching is required on top of the generic decoder patcher. + pass diff --git a/optimum/exporters/openvino/utils.py b/optimum/exporters/openvino/utils.py index de5fd4b355..30bcb2b42a 100644 --- a/optimum/exporters/openvino/utils.py +++ b/optimum/exporters/openvino/utils.py @@ -320,6 +320,7 @@ def _get_kokoro_submodels(model): "internvl_chat", "maira2", "minicpmv", + "youtu_vl", "phi3_v", "qwen2_vl", "qwen2_5_vl", diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 65e3f53f8d..f2b3f8830b 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -5933,6 +5933,243 @@ def generate(self, *args, **kwargs): return super().generate(*args, **kwargs) +class _OVYoutuVLForCausalLM(OVModelForVisualCausalLM): + additional_parts = ["vision_embeddings_merger"] + + def __init__( + self, + language_model: ov.Model, + text_embeddings: ov.Model, + vision_embeddings: ov.Model, + config: PretrainedConfig = None, + device: str = "CPU", + dynamic_shapes: bool = None, + ov_config: Optional[Dict[str, str]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + quantization_config: Union[OVWeightQuantizationConfig, Dict] = None, + **kwargs, + ): + super().__init__( + language_model=language_model, + text_embeddings=text_embeddings, + vision_embeddings=vision_embeddings, + config=config, + device=device, + dynamic_shapes=dynamic_shapes, + ov_config=ov_config, + model_save_dir=model_save_dir, + quantization_config=quantization_config, + **kwargs, + ) + + vision_config = config.vision_config + # vision tower hyper-parameters (mirrors the Siglip2 windowed encoder) + self.spatial_merge_size = 2 + self.spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size + self.patch_size = vision_config.patch_size + self.window_size = self.patch_size * 2 * 8 + self.fullatt_block_indexes = vision_config.fullatt_block_indexes + self.num_vision_layers = vision_config.num_hidden_layers + + class _VisionRope(torch.nn.Module): + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def forward(self, seqlen: int) -> torch.Tensor: + seq = torch.arange(seqlen, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + return torch.outer(seq, self.inv_freq) + + rope_dim = vision_config.hidden_size // vision_config.num_attention_heads // 2 + self._rotary_pos_emb = _VisionRope(rope_dim) + + # mirrors Siglip2Encoder.rot_pos_emb + def rot_pos_emb(self, spatial_shapes): + pos_ids = [] + for h, w in spatial_shapes: + h, w = int(h), int(w) + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + hpos_ids = hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + hpos_ids = hpos_ids.permute(0, 2, 1, 3).flatten() + + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + wpos_ids = wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ) + wpos_ids = wpos_ids.permute(0, 2, 1, 3).flatten() + pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = int(spatial_shapes.max()) + rotary_pos_emb_full = self._rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb + + # mirrors Siglip2Encoder.get_window_index + def get_window_index(self, spatial_shapes): + window_index: list = [] + cu_window_seqlens: list = [0] + window_index_id = 0 + vit_merger_window_size = self.window_size // self.spatial_merge_size // self.patch_size + + for grid_h, grid_w in spatial_shapes: + grid_h, grid_w = int(grid_h), int(grid_w) + grid_t = 1 + llm_grid_h, llm_grid_w = grid_h // self.spatial_merge_size, grid_w // self.spatial_merge_size + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w) + pad_h = (vit_merger_window_size - llm_grid_h % vit_merger_window_size) % vit_merger_window_size + pad_w = (vit_merger_window_size - llm_grid_w % vit_merger_window_size) % vit_merger_window_size + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size + index_padded = torch.nn.functional.pad(index, (0, pad_w, 0, pad_h), "constant", -100) + index_padded = index_padded.reshape( + grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size + ) + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( + grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size + ) + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1) + index_padded = index_padded.reshape(-1) + index_new = index_padded[index_padded != -100] + window_index.append(index_new + window_index_id) + cu_seqlens_tmp = seqlens.cumsum(0) * self.spatial_merge_unit + cu_window_seqlens[-1] + cu_window_seqlens.extend(cu_seqlens_tmp.tolist()) + window_index_id += grid_t * llm_grid_h * llm_grid_w + window_index = torch.cat(window_index, dim=0) + return window_index, cu_window_seqlens + + def forward( + self, + input_ids, + pixel_values=None, + past_key_values=None, + inputs_embeds=None, + attention_mask=None, + position_ids=None, + spatial_shapes=None, + pixel_attention_mask=None, + **kwargs, + ): + return super().forward( + input_ids, + pixel_values=pixel_values, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + position_ids=position_ids, + spatial_shapes=spatial_shapes, + pixel_attention_mask=pixel_attention_mask, + **kwargs, + ) + + def get_vision_embeddings(self, pixel_values, input_ids=None, spatial_shapes=None, **kwargs): + if input_ids is not None and input_ids.shape[1] == 1 and kwargs.get("past_key_values") is not None: + return None + if spatial_shapes is None: + raise ValueError("`spatial_shapes` is required for Youtu-VL vision embeddings.") + if not isinstance(spatial_shapes, torch.Tensor): + spatial_shapes = torch.as_tensor(spatial_shapes) + + hidden_states = torch.from_numpy(self.vision_embeddings(pixel_values).last_hidden_state) + seq_len = hidden_states.shape[0] + + rotary_pos_emb = self.rot_pos_emb(spatial_shapes) + window_index, cu_window_seqlens = self.get_window_index(spatial_shapes) + cu_window_seqlens = torch.tensor(cu_window_seqlens, dtype=torch.int32) + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) + cu_seqlens = torch.repeat_interleave(spatial_shapes[:, 0] * spatial_shapes[:, 1], 1).cumsum( + dim=0, dtype=torch.int32 + ) + cu_seqlens = torch.nn.functional.pad(cu_seqlens, (1, 0), value=0) + + attention_mask = torch.full([1, seq_len, seq_len], torch.finfo(torch.float32).min, dtype=torch.float32) + for i in range(1, len(cu_seqlens)): + attention_mask[..., cu_seqlens[i - 1] : cu_seqlens[i], cu_seqlens[i - 1] : cu_seqlens[i]] = 0 + + window_attention_mask = torch.full( + [1, seq_len, seq_len], torch.finfo(torch.float32).min, dtype=torch.float32 + ) + for i in range(1, len(cu_window_seqlens)): + window_attention_mask[ + ..., cu_window_seqlens[i - 1] : cu_window_seqlens[i], cu_window_seqlens[i - 1] : cu_window_seqlens[i] + ] = 0 + + res = self.vision_embeddings_merger( + pixel_values=hidden_states, + attention_mask=attention_mask, + window_attention_mask=window_attention_mask, + window_index=window_index, + rotary_pos_emb=rotary_pos_emb, + )[0] + return res + + def get_multimodal_embeddings( + self, input_ids, pixel_values=None, attention_mask=None, position_ids=None, **kwargs + ): + inputs_embeds = torch.from_numpy(self.get_text_embeddings(input_ids)) + spatial_shapes = kwargs.pop("spatial_shapes", None) + if pixel_values is not None and input_ids.shape[1] != 1: + image_embeds = self.get_vision_embeddings( + pixel_values, input_ids=input_ids, spatial_shapes=spatial_shapes, **kwargs + ) + if image_embeds is not None: + image_embeds = torch.from_numpy(image_embeds) if isinstance(image_embeds, np.ndarray) else image_embeds + image_token_id = self.config.image_token_id + n_image_tokens = (input_ids == image_token_id).sum().item() + n_image_features = image_embeds.shape[0] + if n_image_tokens != n_image_features: + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, " + f"features {n_image_features}" + ) + mask = (input_ids == image_token_id).unsqueeze(-1).expand_as(inputs_embeds) + image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) + inputs_embeds = inputs_embeds.masked_scatter(mask, image_embeds) + + return inputs_embeds, attention_mask, position_ids + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): + model_inputs = super().prepare_inputs_for_generation(input_ids, past_key_values=past_key_values, **kwargs) + model_inputs["spatial_shapes"] = kwargs.get("spatial_shapes") + model_inputs["pixel_attention_mask"] = kwargs.get("pixel_attention_mask") + return model_inputs + + @staticmethod + def preprocess_inputs( + text: str, + image: Optional["Image"] = None, + processor: Optional[AutoImageProcessor] = None, + tokenizer: Optional[PreTrainedTokenizer] = None, + config: Optional[PretrainedConfig] = None, + video: Optional["VideoInput"] = None, + audio: Optional[np.ndarray] = None, + ): + if processor is None: + raise ValueError("Processor is required.") + if video is not None: + raise ValueError("Video input is not supported") + if audio is not None: + raise ValueError("Audio input is not supported") + if getattr(processor, "chat_template", None) is not None: + content = [{"type": "text", "text": text}] + if image is not None: + content.insert(0, {"type": "image"}) + messages = [{"role": "user", "content": content}] + prompt = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False) + else: + prompt = text + inputs = processor(images=image, text=prompt, return_tensors="pt") + return inputs + + MODEL_TYPE_TO_CLS_MAPPING = { "llava": _OVLlavaForCausalLM, "llava_next": _OVLlavaNextForCausalLM, @@ -5962,4 +6199,5 @@ def generate(self, *args, **kwargs): "qwen3_5_moe_text": _OVQwen3_5ForCausalLM, "minicpmo": _OVMiniCPMOForCausalLM, "videochat_flash_qwen": _OVVideoChatFlashQwenForCausalLM, + "youtu_vl": _OVYoutuVLForCausalLM, } diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 5ba3fb4b19..ea2cf73160 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -814,6 +814,18 @@ class OVCLIExportTestCase(unittest.TestCase): "vision_embeddings_merger_model": {"int8": 12}, }, ), + ( + "image-text-to-text", + "youtu_vl", + 'int4 --group-size 16 --ratio 0.8 --sensitivity-metric "mean_activation_magnitude" ' + "--dataset contextual --num-samples 1 --trust-remote-code", + { + "lm_model": {"int8": 14, "int4": 20}, + "text_embeddings_model": {"int8": 1}, + "vision_embeddings_model": {"int8": 1}, + "vision_embeddings_merger_model": {"int8": 14}, + }, + ), ( "image-text-to-text", "phi4mm", diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index b1542dbfd0..e6641ae5f4 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -1063,6 +1063,7 @@ class OVWeightCompressionTest(unittest.TestCase): (OVModelOpenCLIPForZeroShotImageClassification, "open-clip", False), (OVModelForVisualCausalLM, "llava", False), (OVModelForVisualCausalLM, "qwen2_vl", False), + (OVModelForVisualCausalLM, "youtu_vl", True), ] if is_transformers_version("<", "4.54.0"): diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 70101039ee..a68d12f19b 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -694,6 +694,12 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly if is_transformers_version(">=", "4.51") and is_transformers_version("<", "5"): # SUPPORTED_ARCHITECTURES += ["llama4", "phi4_multimodal"] + SUPPORT_AUDIO.append("phi4mm") + if is_transformers_version(">=", "4.57.0"): + SUPPORTED_ARCHITECTURES += ["youtu_vl"] + if is_transformers_version(">", "4.49"): + SUPPORTED_ARCHITECTURES += ["gemma3", "smolvlm"] + if is_transformers_version(">=", "4.51"): SUPPORTED_ARCHITECTURES += ["llama4"] if is_transformers_version("<", "4.52"): @@ -717,6 +723,8 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): if is_transformers_version(">=", "5.5"): SUPPORTED_ARCHITECTURES += ["gemma4", "gemma4_moe"] + TASK = "image-text-to-text" + REMOTE_CODE_MODELS = ["internvl_chat", "minicpmv", "minicpmo", "llava-qwen2", "phi3_v", "maira2", "phi4mm", "youtu_vl"] if is_transformers_version(">=", "5.10"): SUPPORTED_ARCHITECTURES += ["gemma4_unified"] diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index d9de70648c..8d24fa4ab5 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -358,6 +358,7 @@ def _create_tiny_kokoro_model(): "qwen3_eagle3": "AngelSlim/Qwen3-1.7B_eagle3", "qwen3_vl_eagle3": "optimum-intel-internal-testing/tiny-random-qwen3-vl-eagle3", "videochat_flash_qwen": "optimum-intel-internal-testing/tiny-videochat-flash-qwen", + "youtu_vl": "optimum-intel-internal-testing/tiny-random-youtu-vl", } EAGLE3_MODELS = {"qwen3_eagle3": ("AngelSlim/Qwen3-1.7B_eagle3", "Qwen/Qwen3-1.7B")} @@ -565,6 +566,13 @@ def _create_tiny_kokoro_model(): "decoder": 30, "decoder_with_past": 30, }, + "youtu_vl": { + "lm_model": 34, + "text_embeddings_model": 1, + "vision_embeddings_model": 1, + "vision_embeddings_merger_model": 32, + "vision_embeddings_pos_model": 1, + }, } TEST_IMAGE_URL = "http://images.cocodataset.org/val2017/000000039769.jpg" From 8bea69c5ddac2c6371f511a8dc18c6b6ce7bfc53 Mon Sep 17 00:00:00 2001 From: Mohamed-Ashraf273 Date: Thu, 2 Jul 2026 18:47:38 +0300 Subject: [PATCH 2/2] add tests --- optimum/exporters/openvino/model_configs.py | 46 +++++----- tests/openvino/test_seq2seq.py | 1 + tests/openvino/utils_tests.py | 97 ++++++++++++++++++++- 3 files changed, 121 insertions(+), 23 deletions(-) diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 936a1e3553..b622f411b6 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -3887,7 +3887,9 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int @register_in_tasks_manager("youtu_vl", *["image-text-to-text"], library_name="transformers") class YoutuVLOpenVINOConfig(BaseVLMOpenVINOConfig): - SUPPORTED_BEHAVIORS = [model_type.value for model_type in Qwen2VLConfigBehavior] + SUPPORTED_BEHAVIORS = [ + model_type.value for model_type in QwenVLConfigBehavior if model_type != QwenVLConfigBehavior.VISION_EMBEDDINGS_POS + ] NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig DUMMY_INPUT_GENERATOR_CLASSES = (YoutuVLVisionEmbeddingsDummyInputGenerator,) MIN_TRANSFORMERS_VERSION = "4.49.0" @@ -3898,7 +3900,7 @@ def __init__( task: str = "feature-extraction", int_dtype: str = "int64", float_dtype: str = "fp32", - behavior: Qwen2VLConfigBehavior = Qwen2VLConfigBehavior.VISION_EMBEDDINGS, + behavior: QwenVLConfigBehavior = QwenVLConfigBehavior.VISION_EMBEDDINGS, preprocessors: Optional[List[Any]] = None, **kwargs, ): @@ -3912,49 +3914,49 @@ def __init__( self._behavior = behavior self._orig_config = config if ( - self._behavior in [Qwen2VLConfigBehavior.VISION_EMBEDDINGS, Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER] + self._behavior in [QwenVLConfigBehavior.VISION_EMBEDDINGS, QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER] and hasattr(config, "vision_config") ): self._config = config.vision_config self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config) @staticmethod - def get_model_for_behavior(model, behavior: Union[str, Qwen2VLConfigBehavior]): - if isinstance(behavior, str) and not isinstance(behavior, Qwen2VLConfigBehavior): - behavior = Qwen2VLConfigBehavior(behavior) + def get_model_for_behavior(model, behavior: Union[str, QwenVLConfigBehavior]): + if isinstance(behavior, str) and not isinstance(behavior, QwenVLConfigBehavior): + behavior = QwenVLConfigBehavior(behavior) - if behavior == Qwen2VLConfigBehavior.LANGUAGE: + if behavior == QwenVLConfigBehavior.LANGUAGE: return model - if behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS: # patch embedding (Siglip2VisionEmbeddingsWoPos) vision_embeddings = model.siglip2.vision_model.embeddings vision_embeddings.config = model.config.vision_config return vision_embeddings - if behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + if behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER: # the encoder + post layernorm + merger are exported together; they are # spread across several submodules of the top-level model, so we export the # whole model and rely on the patcher to wire the right forward. model.config.vision_config.fullatt_block_indexes = model.config.vision_config.fullatt_block_indexes return model - if behavior == Qwen2VLConfigBehavior.TEXT_EMBEDDINGS: + if behavior == QwenVLConfigBehavior.TEXT_EMBEDDINGS: text_embedding = model.get_input_embeddings() text_embedding.config = model.config return text_embedding def with_behavior( self, - behavior: Union[str, Qwen2VLConfigBehavior], + behavior: Union[str, QwenVLConfigBehavior], ): - if isinstance(behavior, str) and not isinstance(behavior, Qwen2VLConfigBehavior): - behavior = Qwen2VLConfigBehavior(behavior) + if isinstance(behavior, str) and not isinstance(behavior, QwenVLConfigBehavior): + behavior = QwenVLConfigBehavior(behavior) - if behavior == Qwen2VLConfigBehavior.TEXT_EMBEDDINGS: + if behavior == QwenVLConfigBehavior.TEXT_EMBEDDINGS: return get_vlm_text_embeddings_config("youtu_vl", self._orig_config, self.int_dtype, self.float_dtype) - if behavior == Qwen2VLConfigBehavior.LANGUAGE: + if behavior == QwenVLConfigBehavior.LANGUAGE: return get_vlm_text_generation_config( "youtu_vl", self._orig_config, @@ -3963,7 +3965,7 @@ def with_behavior( model_patcher=YoutuVLLanguageModelPatcher, ) - if behavior in [Qwen2VLConfigBehavior.VISION_EMBEDDINGS, Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER]: + if behavior in [QwenVLConfigBehavior.VISION_EMBEDDINGS, QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER]: return self.__class__( self._orig_config, task=self.task, @@ -3975,17 +3977,17 @@ def with_behavior( def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[Dict[str, Any]] = None): model_kwargs = model_kwargs or {} - if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER: return YoutuVLVisionEmbMergerPatcher(self, model, model_kwargs) - if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS: return ModelPatcher(self, model, model_kwargs=model_kwargs) return super().patch_model_for_export(model, model_kwargs) @property def inputs(self) -> Dict[str, Dict[int, str]]: - if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS: + if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS: return {"pixel_values": {0: "batch_size", 1: "num_patches"}} - if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER: + if self._behavior == QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER: return { "hidden_states": {0: "sequence_length"}, "attention_mask": {1: "sequence_length", 2: "sequence_length"}, @@ -3998,8 +4000,8 @@ def inputs(self) -> Dict[str, Dict[int, str]]: @property def outputs(self) -> Dict[str, Dict[int, str]]: if self._behavior in [ - Qwen2VLConfigBehavior.VISION_EMBEDDINGS, - Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER, + QwenVLConfigBehavior.VISION_EMBEDDINGS, + QwenVLConfigBehavior.VISION_EMBEDDINGS_MERGER, ]: return {"last_hidden_state": {0: "seq_len"}} return {} diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index a68d12f19b..1db49528a5 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -746,6 +746,7 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): "maira2", "phi4mm", "videochat_flash_qwen", + "youtu_vl", ] IMAGE = Image.open( requests.get( diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 8d24fa4ab5..931853bce1 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -142,6 +142,100 @@ def _create_tiny_kokoro_model(): return str(output_dir) +def _create_tiny_youtu_vl_model(): + """Generate and cache a tiny random Youtu-VL model for tests.""" + import shutil + + from huggingface_hub import snapshot_download + from transformers import AutoConfig, AutoModelForCausalLM + + model_id = "tencent/Youtu-VL-4B-Instruct" + output_dir = Path(tempfile.gettempdir()) / "optimum_intel_tiny_random_youtu_vl" + config_file = output_dir / "config.json" + weights_file = output_dir / "model.safetensors" + if config_file.exists() and weights_file.exists(): + with open(config_file, encoding="utf-8") as f: + cached_config = json.load(f) + if ( + cached_config.get("dtype") == "float32" + and cached_config.get("vision_config", {}).get("dtype") == "float32" + ): + return str(output_dir) + + repo_path = Path( + snapshot_download( + model_id, + token=os.environ.get("HF_TOKEN"), + ignore_patterns=["*.safetensors", "*.bin", "*.pt", "*.gguf", "model.safetensors.index.json"], + ) + ) + if output_dir.exists(): + shutil.rmtree(output_dir) + shutil.copytree(repo_path, output_dir) + + # Segmentation dependencies are optional for model export and are not + # available in every test environment. + modeling_file = output_dir / "modeling_youtu_vl.py" + modeling_source = modeling_file.read_text(encoding="utf-8") + modeling_source = modeling_source.replace( + "import pydensecrf.densecrf as dcrf\nfrom pydensecrf.utils import unary_from_softmax", + "try:\n" + " import pydensecrf.densecrf as dcrf\n" + " from pydensecrf.utils import unary_from_softmax\n" + "except ImportError:\n" + " dcrf = None\n" + " unary_from_softmax = None", + ) + modeling_file.write_text(modeling_source, encoding="utf-8") + + with open(repo_path / "config.json", encoding="utf-8") as f: + config_dict = json.load(f) + + config_dict.update( + { + "hidden_size": 64, + "intermediate_size": 128, + "kv_lora_rank": 16, + "max_position_embeddings": 4096, + "n_group": 1, + "n_routed_experts": 4, + "n_shared_experts": 1, + "num_attention_heads": 1, + "num_experts_per_tok": 2, + "num_hidden_layers": 2, + "num_key_value_heads": 1, + "q_lora_rank": 32, + "tie_word_embeddings": True, + "topk_group": 1, + "dtype": "float32", + "torch_dtype": "float32", + "use_cache": False, + } + ) + config_dict["vision_config"].update( + { + "fullatt_block_indexes": [1], + "hidden_size": 32, + "intermediate_size": 64, + "num_attention_heads": 1, + "num_hidden_layers": 2, + "num_patches": 256, + "out_hidden_size": 64, + "window_size": 16, + "dtype": "float32", + } + ) + with open(config_file, "w", encoding="utf-8") as f: + json.dump(config_dict, f, indent=2) + + config = AutoConfig.from_pretrained(output_dir, trust_remote_code=True) + model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) + model.eval() + model.save_pretrained(output_dir) + + return str(output_dir) + + SEED = 42 F32_CONFIG = {"INFERENCE_PRECISION_HINT": "f32"} @@ -358,7 +452,7 @@ def _create_tiny_kokoro_model(): "qwen3_eagle3": "AngelSlim/Qwen3-1.7B_eagle3", "qwen3_vl_eagle3": "optimum-intel-internal-testing/tiny-random-qwen3-vl-eagle3", "videochat_flash_qwen": "optimum-intel-internal-testing/tiny-videochat-flash-qwen", - "youtu_vl": "optimum-intel-internal-testing/tiny-random-youtu-vl", + "youtu_vl": _create_tiny_youtu_vl_model(), } EAGLE3_MODELS = {"qwen3_eagle3": ("AngelSlim/Qwen3-1.7B_eagle3", "Qwen/Qwen3-1.7B")} @@ -602,6 +696,7 @@ def _create_tiny_kokoro_model(): "qwen3_vl_eagle3", "qwen3_asr", "videochat_flash_qwen", + "youtu_vl", ) if is_transformers_version("<", "5"):