From e6df5a0f7d47d0d02a94e4295edd0f2c8715ed4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 15 Apr 2026 09:47:01 +0200 Subject: [PATCH 01/87] transformers v5.5 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f9efdf679..81a2d644bd 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@transformers-v5", - "transformers>=4.45,<5.1", + "transformers>=4.45,<5.6", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From 4fcd786c6ca95f9b0feafe8c82f00080e48f2891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 15 Apr 2026 18:13:17 +0200 Subject: [PATCH 02/87] update setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 81a2d644bd..b8020f9c3f 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", - "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@transformers-v5", + "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", "transformers>=4.45,<5.6", "setuptools", "huggingface-hub>=0.23.2,<2.0", From 9b439b49eb2030cf71e46412262f1923cf64ea49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 15 Apr 2026 18:32:11 +0200 Subject: [PATCH 03/87] set min required transformers verion to v4.57 --- .github/workflows/test_openvino.yml | 2 +- .github/workflows/test_openvino_slow.yml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index ba60fc597a..dc31fefd85 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -38,7 +38,7 @@ jobs: "*diffusion*", "*quantization*", ] - transformers-version: ["4.45.0", "4.57.6", "latest"] + transformers-version: ["4.57.6", "latest"] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test_openvino_slow.yml b/.github/workflows/test_openvino_slow.yml index 99603edd27..731388490a 100644 --- a/.github/workflows/test_openvino_slow.yml +++ b/.github/workflows/test_openvino_slow.yml @@ -36,7 +36,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04", "windows-2022"] - transformers-version: ["4.45.0", "latest"] + transformers-version: ["4.57.6", "latest"] include: - transformers-version: "main" os: "ubuntu-22.04" diff --git a/setup.py b/setup.py index b8020f9c3f..73ab45f19c 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", - "transformers>=4.45,<5.6", + "transformers>=4.57,<5.3", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From 88d4f1ac5f5cead137411cf2ec5fe0ed054d0600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 16 Apr 2026 16:53:19 +0200 Subject: [PATCH 04/87] fix position ids generation --- optimum/intel/openvino/modeling_decoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index ccf177df9d..e26a354568 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -39,7 +39,7 @@ from ...exporters.openvino import ensure_stateful_is_available, main_export, patch_stateful from ...exporters.openvino.stateful import model_has_state from ...exporters.openvino.utils import SSM_MODELS -from ..utils.import_utils import compare_versions +from ..utils.import_utils import compare_versions, is_transformers_version from ..utils.modeling_utils import MULTI_QUERY_ATTN_MODELS from .configuration import ( OVConfig, @@ -668,7 +668,8 @@ def _update_model_kwargs_for_generation( outputs=outputs, model_kwargs=model_kwargs, is_encoder_decoder=is_encoder_decoder, **kwargs ) - if "position_ids" in model_kwargs: + # _prepare_position_ids_for_generation will infer position ids since transformers v5.2 + if "position_ids" in model_kwargs and is_transformers_version("<", "5.2"): position_ids = model_kwargs["position_ids"] new_position_id = position_ids[..., -1:].clone() new_position_id += 1 From c31a88b3c16e4e03537f08a737fb18fcdcba29aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 16 Apr 2026 18:03:51 +0200 Subject: [PATCH 05/87] convbert --- tests/openvino/test_modeling.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 372cd28943..aff5bab49d 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -1113,7 +1113,6 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): "albert", "bert", "camembert", - "convbert", "deberta", "deberta-v2", "distilbert", @@ -1131,13 +1130,16 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): ) # accuracy issue, need additional investigation - if is_transformers_version("<", "4.51.0"): + if is_transformers_version("<", "4.51"): SUPPORTED_ARCHITECTURES += ("nystromformer",) # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly if is_transformers_version("<", "5"): SUPPORTED_ARCHITECTURES += ("data2vec-text", "flaubert", "xlm") + if is_transformers_version("!=", "4.52"): + SUPPORTED_ARCHITECTURES += ("convbert",) + @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] From da7c4108d2fcc434bc20bedeb3d69cde13eb0553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 17 Apr 2026 17:18:46 +0200 Subject: [PATCH 06/87] fix position_ids generation for qwenvl models --- optimum/intel/openvino/modeling_visual_language.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index beb7b974eb..806c2b97df 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -190,7 +190,7 @@ def prepare_inputs( if past_len: position_ids = position_ids[:, -inputs_embeds.shape[1] :] - if (self.config.model_type in ["qwen2_vl", "qwen3_vl"]) and position_ids.ndim != 3: + if (self.config.model_type in ["qwen2_vl", "qwen2_5_vl", "qwen3_vl"]) and position_ids.ndim != 3: position_ids = np.repeat(np.expand_dims(position_ids, 0), 3, axis=0) inputs["position_ids"] = position_ids @@ -2895,7 +2895,11 @@ def get_multimodal_embeddings( inputs_embeds[video_mask] = video_embeds # if we get 4D attention mask we cannot calculate rope deltas anymore. - if position_ids is None and input_ids is not None and (attention_mask is None or attention_mask.ndim == 2): + if ( + (position_ids is None or position_ids.ndim < 3) + and input_ids is not None + and (attention_mask is None or attention_mask.ndim == 2) + ): # calculate RoPE index once per generation in the pre-fill stage only if (cache_position is not None and cache_position[0] == 0) or self.rope_deltas is None: position_ids, rope_deltas = self.get_rope_index( @@ -3286,7 +3290,11 @@ def get_multimodal_embeddings( inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) # if we get 4D attention mask we cannot calculate rope deltas anymore. - if position_ids is None and input_ids is not None and (attention_mask is None or attention_mask.ndim == 2): + if ( + (position_ids is None or position_ids.ndim < 3) + and input_ids is not None + and (attention_mask is None or attention_mask.ndim == 2) + ): # calculate RoPE index once per generation in the pre-fill stage only if (cache_position is not None and cache_position[0] == 0) or self.rope_deltas is None: position_ids, rope_deltas = self.get_rope_index( From 5ca59b82b16dc113a590ef705fb0d38ccacd12f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 17 Apr 2026 17:19:06 +0200 Subject: [PATCH 07/87] check for _prepare_position_ids_for_generation --- optimum/intel/openvino/modeling_decoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index e26a354568..85b109268c 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -39,7 +39,7 @@ from ...exporters.openvino import ensure_stateful_is_available, main_export, patch_stateful from ...exporters.openvino.stateful import model_has_state from ...exporters.openvino.utils import SSM_MODELS -from ..utils.import_utils import compare_versions, is_transformers_version +from ..utils.import_utils import compare_versions from ..utils.modeling_utils import MULTI_QUERY_ATTN_MODELS from .configuration import ( OVConfig, @@ -669,7 +669,7 @@ def _update_model_kwargs_for_generation( ) # _prepare_position_ids_for_generation will infer position ids since transformers v5.2 - if "position_ids" in model_kwargs and is_transformers_version("<", "5.2"): + if "position_ids" in model_kwargs and not hasattr(self, "_prepare_position_ids_for_generation"): position_ids = model_kwargs["position_ids"] new_position_id = position_ids[..., -1:].clone() new_position_id += 1 From 661f0a58072bd0df1ee0de0e66cdbd58471ee0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 17 Apr 2026 17:38:08 +0200 Subject: [PATCH 08/87] fix qwen3vl --- optimum/intel/openvino/modeling_visual_language.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 806c2b97df..791e7a6430 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -3716,7 +3716,7 @@ def get_multimodal_embeddings( visual_pos_masks = video_mask deepstack_visual_embeds = deepstack_video_embeds - if position_ids is None: + if position_ids is None or position_ids.ndim < 3: attention_mask_tensor = ( attention_mask if not isinstance(attention_mask, dict) else attention_mask["full_attention"] ) From b52550f7a76ba8ea8ab4e951f39bd6f4d24ab4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 20 Apr 2026 16:19:07 +0200 Subject: [PATCH 09/87] fix transformers version --- tests/openvino/test_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index aff5bab49d..ce03881c36 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -1137,7 +1137,7 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): if is_transformers_version("<", "5"): SUPPORTED_ARCHITECTURES += ("data2vec-text", "flaubert", "xlm") - if is_transformers_version("!=", "4.52"): + if is_transformers_version("!=", "5.2"): SUPPORTED_ARCHITECTURES += ("convbert",) @parameterized.expand(SUPPORTED_ARCHITECTURES) From c5f9c470025caaa53f3b256bee44717bf680a21b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 20 Apr 2026 16:19:46 +0200 Subject: [PATCH 10/87] transformers v5.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73ab45f19c..12ff9e0391 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", - "transformers>=4.57,<5.3", + "transformers>=4.57,<5.4", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From ee941ff701cd65af6cedd0bf9bcb0920199a8bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 20 Apr 2026 16:34:33 +0200 Subject: [PATCH 11/87] datasets test extra setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12ff9e0391..f5c42220ac 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "sentence-transformers<5.4.0", "open_clip_torch>=2.26.1", "peft", - "datasets>=1.4.0,<4.0.0", + "datasets>=1.4.0", "tbb", "langchain-huggingface", "hf_xet", From 4a77bce1a16d5bd8990d893fd79e8ff679e3d0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 22 Apr 2026 11:23:00 +0200 Subject: [PATCH 12/87] create attention_mask when needed as not created in generate since v5.2 --- optimum/intel/openvino/modeling_seq2seq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index cb8d6b7fa4..7d9f0dceef 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -875,6 +875,8 @@ def forward( # Add the attention_mask inputs when needed if "attention_mask" in self.input_names: + if attention_mask is None: + attention_mask = torch.ones_like(inputs[self.main_input_name]) inputs["attention_mask"] = attention_mask # Run inference From 06187d017a24f4757a609b718742fc96344188c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 23 Apr 2026 17:25:14 +0200 Subject: [PATCH 13/87] fix mm_token_type_ids --- .../openvino/modeling_visual_language.py | 447 ++++++------------ 1 file changed, 133 insertions(+), 314 deletions(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 791e7a6430..a713c465cb 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -26,23 +26,29 @@ PreTrainedTokenizer, ) from transformers.modeling_outputs import BaseModelOutputWithPooling -from transformers.models.qwen2_vl.modeling_qwen2_vl import VisionRotaryEmbedding +from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel +from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel, VisionRotaryEmbedding +from transformers.models.qwen3_vl.modeling_qwen3_vl import ( + Qwen3VLModel, + Qwen3VLVisionModel, + Qwen3VLVisionRotaryEmbedding, +) from transformers.utils import ModelOutput -from ...exporters.openvino import main_export -from ...exporters.openvino.stateful import ensure_stateful_is_available, model_has_input_output_name -from ...exporters.openvino.utils import save_config -from ..utils.import_utils import is_transformers_version -from .configuration import OVConfig, OVQuantizationConfigBase, OVWeightQuantizationConfig -from .modeling_base import OVBaseModel, OVModelPart -from .modeling_decoder import CausalLMOutputWithPast, OVModelForCausalLM -from .utils import ( +from optimum.exporters.openvino import main_export +from optimum.exporters.openvino.stateful import ensure_stateful_is_available, model_has_input_output_name +from optimum.exporters.openvino.utils import save_config +from optimum.intel.openvino.configuration import OVConfig, OVQuantizationConfigBase, OVWeightQuantizationConfig +from optimum.intel.openvino.modeling_base import OVBaseModel, OVModelPart +from optimum.intel.openvino.modeling_decoder import CausalLMOutputWithPast, OVModelForCausalLM +from optimum.intel.openvino.utils import ( OV_LANGUAGE_MODEL_NAME, OV_TEXT_EMBEDDINGS_MODEL_NAME, OV_VISION_EMBEDDINGS_MODEL_NAME, TemporaryDirectory, classproperty, ) +from optimum.intel.utils.import_utils import is_transformers_version if is_transformers_version(">=", "4.46.0"): @@ -2570,6 +2576,7 @@ class QWen2VLModelOutputWithPast(ModelOutput): class _OVQwen2VLForCausalLM(OVModelForVisualCausalLM): + get_rope_index = Qwen2VLModel.get_rope_index additional_parts = ["vision_embeddings_merger"] def __init__( @@ -2615,8 +2622,23 @@ def prepare_inputs_for_generation( pixel_values_videos=None, image_grid_thw=None, video_grid_thw=None, + mm_token_type_ids=None, + is_first_iteration=False, + next_sequence_length=None, **kwargs, ): + # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 + if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if next_sequence_length is not None: + past_len = input_ids.shape[1] - next_sequence_length + cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) + elif not is_first_iteration and attention_mask is not None: + # v5.3 decode step: input_ids is already sliced to 1 token, use attention_mask length + past_len = attention_mask.shape[1] - 1 + cache_position = torch.tensor([past_len], device=input_ids.device) + else: + cache_position = torch.arange(input_ids.shape[1], device=input_ids.device) + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model if past_key_values is not None: if inputs_embeds is not None and input_ids.shape[1] == 0: # Exception 4 @@ -2647,6 +2669,7 @@ def prepare_inputs_for_generation( "image_grid_thw": image_grid_thw, "video_grid_thw": video_grid_thw, "cache_position": cache_position, + "mm_token_type_ids": mm_token_type_ids, } ) return model_inputs @@ -2671,156 +2694,6 @@ def _update_model_kwargs_for_generation( return model_kwargs - # Copied from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L1423 - def get_rope_index( - self, - input_ids: Optional[torch.LongTensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Calculate the 3D rope index based on image and video's temporal, height and width in LLM. - - Explanation: - Each embedding sequence contains vision embedding and text embedding or just contains text embedding. - - For pure text embedding sequence, the rotary position embedding has no difference with modern LLMs. - Examples: - input_ids: [T T T T T], here T is for text. - temporal position_ids: [0, 1, 2, 3, 4] - height position_ids: [0, 1, 2, 3, 4] - width position_ids: [0, 1, 2, 3, 4] - - For vision and text embedding sequence, we calculate 3D rotary position embedding for vision part - and 1D rotary position embedding for text part. - Examples: - Assume we have a video input with 3 temporal patches, 2 height patches and 2 width patches. - input_ids: [V V V V V V V V V V V V T T T T T], here V is for vision. - vision temporal position_ids: [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] - vision height position_ids: [0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1] - vision width position_ids: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] - text temporal position_ids: [3, 4, 5, 6, 7] - text height position_ids: [3, 4, 5, 6, 7] - text width position_ids: [3, 4, 5, 6, 7] - Here we calculate the text start position_ids as the max vision position_ids plus 1. - - Args: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - image_grid_thw (`torch.LongTensor` of shape `(num_images, 3)`, *optional*): - The temporal, height and width of feature shape of each image in LLM. - video_grid_thw (`torch.LongTensor` of shape `(num_videos, 3)`, *optional*): - The temporal, height and width of feature shape of each video in LLM. - attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - Returns: - position_ids (`torch.LongTensor` of shape `(3, batch_size, sequence_length)`) - mrope_position_deltas (`torch.Tensor` of shape `(batch_size)`) - """ - spatial_merge_size = self.config.vision_config.spatial_merge_size - image_token_id = self.config.image_token_id - video_token_id = self.config.video_token_id - vision_start_token_id = self.config.vision_start_token_id - mrope_position_deltas = [] - if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): - total_input_ids = input_ids - if attention_mask is None: - attention_mask = torch.ones_like(total_input_ids) - position_ids = torch.ones( - 3, input_ids.shape[0], input_ids.shape[1], dtype=input_ids.dtype, device=input_ids.device - ) - image_index, video_index = 0, 0 - for i, input_ids in enumerate(total_input_ids): - input_ids = input_ids[attention_mask[i].to(input_ids.device) == 1] - image_nums, video_nums = 0, 0 - vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) - vision_tokens = input_ids[vision_start_indices + 1] - image_nums = (vision_tokens == image_token_id).sum() - video_nums = (vision_tokens == video_token_id).sum() - input_tokens = input_ids.tolist() - llm_pos_ids_list: list = [] - st = 0 - remain_images, remain_videos = image_nums, video_nums - for _ in range(image_nums + video_nums): - if image_token_id in input_tokens and remain_images > 0: - ed_image = input_tokens.index(image_token_id, st) - else: - ed_image = len(input_tokens) + 1 - if video_token_id in input_tokens and remain_videos > 0: - ed_video = input_tokens.index(video_token_id, st) - else: - ed_video = len(input_tokens) + 1 - if ed_image < ed_video: - t, h, w = ( - image_grid_thw[image_index][0], - image_grid_thw[image_index][1], - image_grid_thw[image_index][2], - ) - image_index += 1 - remain_images -= 1 - ed = ed_image - else: - t, h, w = ( - video_grid_thw[video_index][0], - video_grid_thw[video_index][1], - video_grid_thw[video_index][2], - ) - video_index += 1 - remain_videos -= 1 - ed = ed_video - llm_grid_t, llm_grid_h, llm_grid_w = ( - t.item(), - h.item() // spatial_merge_size, - w.item() // spatial_merge_size, - ) - text_len = ed - st - - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - t_index = torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w).flatten() - h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() - w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() - llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) - st = ed + llm_grid_t * llm_grid_h * llm_grid_w - - if st < len(input_tokens): - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - text_len = len(input_tokens) - st - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) - position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) - mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) - mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) - return position_ids, mrope_position_deltas - else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) - max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], - device=input_ids.device, - dtype=input_ids.dtype, - ) - - return position_ids, mrope_position_deltas - def get_vision_embeddings(self, pixel_values, grid_thw, **kwargs): hidden_states = self.vision_embeddings(pixel_values)[0] rotary_pos_emb = self.rot_pos_emb(grid_thw) @@ -2902,9 +2775,28 @@ def get_multimodal_embeddings( ): # calculate RoPE index once per generation in the pre-fill stage only if (cache_position is not None and cache_position[0] == 0) or self.rope_deltas is None: - position_ids, rope_deltas = self.get_rope_index( - input_ids, image_grid_thw, video_grid_thw, attention_mask - ) + if is_transformers_version(">=", "5.3.0"): + # since transformers v5.3, get_rope_index requires mm_token_type_ids + mm_token_type_ids = kwargs.get("mm_token_type_ids") + if mm_token_type_ids is None: + mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int32) + mm_token_type_ids[input_ids == self.config.image_token_id] = 1 + mm_token_type_ids[input_ids == self.config.video_token_id] = 2 + position_ids, rope_deltas = self.get_rope_index( + input_ids, + mm_token_type_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + ) + else: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + attention_mask, + ) + self.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids else: @@ -2991,6 +2883,9 @@ def preprocess_inputs( class _OVQwen2_5_VLForCausalLM(OVModelForVisualCausalLM): + get_rope_index = Qwen2_5_VLModel.get_rope_index + get_vision_position_ids = getattr(Qwen2_5_VLModel, "get_vision_position_ids", None) + additional_parts = ["vision_embeddings_merger"] def __init__( @@ -3034,134 +2929,6 @@ def forward(self, seqlen: int) -> torch.Tensor: head_dim = config.vision_config.hidden_size // config.vision_config.num_heads self._rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2) - def get_rope_index( - self, - input_ids: Optional[torch.LongTensor] = None, - image_grid_thw: Optional[torch.LongTensor] = None, - video_grid_thw: Optional[torch.LongTensor] = None, - second_per_grid_ts: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - # modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L1546 - """ - Calculate the 3D rope index based on image and video's temporal, height and width in LLM. - """ - spatial_merge_size = self.config.vision_config.spatial_merge_size - image_token_id = self.config.image_token_id - video_token_id = self.config.video_token_id - vision_start_token_id = self.config.vision_start_token_id - mrope_position_deltas = [] - if input_ids is not None and (image_grid_thw is not None or video_grid_thw is not None): - total_input_ids = input_ids - if attention_mask is None: - attention_mask = torch.ones_like(total_input_ids) - position_ids = torch.ones( - 3, - input_ids.shape[0], - input_ids.shape[1], - dtype=input_ids.dtype, - device=input_ids.device, - ) - image_index, video_index = 0, 0 - attention_mask = attention_mask.to(total_input_ids.device) - for i, input_ids in enumerate(total_input_ids): - input_ids = input_ids[attention_mask[i] == 1] - image_nums, video_nums = 0, 0 - vision_start_indices = torch.argwhere(input_ids == vision_start_token_id).squeeze(1) - vision_tokens = input_ids[vision_start_indices + 1] - image_nums = (vision_tokens == image_token_id).sum() - video_nums = (vision_tokens == video_token_id).sum() - input_tokens = input_ids.tolist() - llm_pos_ids_list: list = [] - st = 0 - remain_images, remain_videos = image_nums, video_nums - for _ in range(image_nums + video_nums): - if image_token_id in input_tokens and remain_images > 0: - ed_image = input_tokens.index(image_token_id, st) - else: - ed_image = len(input_tokens) + 1 - if video_token_id in input_tokens and remain_videos > 0: - ed_video = input_tokens.index(video_token_id, st) - else: - ed_video = len(input_tokens) + 1 - if ed_image < ed_video: - t, h, w = ( - image_grid_thw[image_index][0], - image_grid_thw[image_index][1], - image_grid_thw[image_index][2], - ) - second_per_grid_t = 0 - image_index += 1 - remain_images -= 1 - ed = ed_image - - else: - t, h, w = ( - video_grid_thw[video_index][0], - video_grid_thw[video_index][1], - video_grid_thw[video_index][2], - ) - if second_per_grid_ts is not None: - second_per_grid_t = second_per_grid_ts[video_index] - else: - second_per_grid_t = 1.0 - video_index += 1 - remain_videos -= 1 - ed = ed_video - llm_grid_t, llm_grid_h, llm_grid_w = ( - t.item(), - h.item() // spatial_merge_size, - w.item() // spatial_merge_size, - ) - text_len = ed - st - - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - range_tensor = torch.arange(llm_grid_t).view(-1, 1) - expanded_range = range_tensor.expand(-1, llm_grid_h * llm_grid_w) - - time_tensor = expanded_range * second_per_grid_t * self.config.vision_config.tokens_per_second - - time_tensor_long = time_tensor.long() - t_index = time_tensor_long.flatten() - - h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(llm_grid_t, -1, llm_grid_w).flatten() - w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(llm_grid_t, llm_grid_h, -1).flatten() - llm_pos_ids_list.append(torch.stack([t_index, h_index, w_index]) + text_len + st_idx) - st = ed + llm_grid_t * llm_grid_h * llm_grid_w - - if st < len(input_tokens): - st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0 - text_len = len(input_tokens) - st - llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx) - - llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) - position_ids[..., i, attention_mask[i] == 1] = llm_positions.to(position_ids.device) - mrope_position_deltas.append(llm_positions.max() + 1 - len(total_input_ids[i])) - mrope_position_deltas = torch.tensor(mrope_position_deltas, device=input_ids.device).unsqueeze(1) - return position_ids, mrope_position_deltas - else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) - max_position_ids = position_ids.max(0, keepdim=False)[0].max(-1, keepdim=True)[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], - device=input_ids.device, - dtype=input_ids.dtype, - ) - - return position_ids, mrope_position_deltas - def prepare_inputs_for_generation( self, input_ids, @@ -3176,8 +2943,23 @@ def prepare_inputs_for_generation( image_grid_thw=None, video_grid_thw=None, second_per_grid_ts=None, + mm_token_type_ids=None, + is_first_iteration=False, + next_sequence_length=None, **kwargs, ): + # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 + if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if next_sequence_length is not None: + past_len = input_ids.shape[1] - next_sequence_length + cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) + elif not is_first_iteration and attention_mask is not None: + # v5.3 decode step: input_ids is already sliced to 1 token, use attention_mask length + past_len = attention_mask.shape[1] - 1 + cache_position = torch.tensor([past_len], device=input_ids.device) + else: + cache_position = torch.arange(input_ids.shape[1], device=input_ids.device) + if past_key_values is not None: if inputs_embeds is not None and input_ids.shape[1] == 0: inputs_embeds = inputs_embeds[:, -cache_position.shape[0] :] @@ -3208,6 +2990,7 @@ def prepare_inputs_for_generation( "video_grid_thw": video_grid_thw, "cache_position": cache_position, "second_per_grid_ts": second_per_grid_ts, + "mm_token_type_ids": mm_token_type_ids, } ) return model_inputs @@ -3297,9 +3080,29 @@ def get_multimodal_embeddings( ): # calculate RoPE index once per generation in the pre-fill stage only if (cache_position is not None and cache_position[0] == 0) or self.rope_deltas is None: - position_ids, rope_deltas = self.get_rope_index( - input_ids, image_grid_thw, video_grid_thw, second_per_grid_ts, attention_mask - ) + if is_transformers_version(">=", "5.3.0"): + # since transformers v5.3, get_rope_index requires mm_token_type_ids + mm_token_type_ids = kwargs.get("mm_token_type_ids") + if mm_token_type_ids is None: + mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int32) + mm_token_type_ids[input_ids == self.config.image_token_id] = 1 + mm_token_type_ids[input_ids == self.config.video_token_id] = 2 + position_ids, rope_deltas = self.get_rope_index( + input_ids, + mm_token_type_ids, + image_grid_thw, + video_grid_thw, + second_per_grid_ts, + attention_mask, + ) + else: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + second_per_grid_ts, + attention_mask, + ) self.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids else: @@ -3433,21 +3236,6 @@ def preprocess_inputs( return inputs -if is_transformers_version(">=", "4.57.0"): - from transformers.models.qwen3_vl.modeling_qwen3_vl import ( - Qwen3VLModel, - Qwen3VLVisionModel, - Qwen3VLVisionRotaryEmbedding, - ) -else: - - class Qwen3VLModel: - pass - - class Qwen3VLVisionModel: - pass - - # The inheritance from Qwen3VLModel is needed to get access to methods: # get_placeholder_mask(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L1066 # get_rope_index(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L916 @@ -3513,8 +3301,23 @@ def prepare_inputs_for_generation( pixel_values_videos=None, image_grid_thw=None, video_grid_thw=None, + mm_token_type_ids=None, + is_first_iteration=False, + next_sequence_length=None, **kwargs, ): + # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 + if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if next_sequence_length is not None: + past_len = input_ids.shape[1] - next_sequence_length + cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) + elif not is_first_iteration and attention_mask is not None: + # v5.3 decode step: input_ids is already sliced to 1 token, use attention_mask length + past_len = attention_mask.shape[1] - 1 + cache_position = torch.tensor([past_len], device=input_ids.device) + else: + cache_position = torch.arange(input_ids.shape[1], device=input_ids.device) + # Overwritten -- in specific circumstances we don't want to forward image inputs to the model if past_key_values is not None: if inputs_embeds is not None and input_ids.shape[1] == 0: # Exception 4 @@ -3545,6 +3348,7 @@ def prepare_inputs_for_generation( "image_grid_thw": image_grid_thw, "video_grid_thw": video_grid_thw, "cache_position": cache_position, + "mm_token_type_ids": mm_token_type_ids, } ) return model_inputs @@ -3732,12 +3536,27 @@ def get_multimodal_embeddings( # It is safe to assume that `length!=1` means we're in pre-fill because compiled # models currently cannot do asssisted decoding if self.rope_deltas is None: - position_ids, rope_deltas = self.get_rope_index( - input_ids, - image_grid_thw, - video_grid_thw, - attention_mask=attention_mask_tensor, - ) + if is_transformers_version(">=", "5.3.0"): + # since transformers v5.3, get_rope_index requires mm_token_type_ids + mm_token_type_ids = kwargs.get("mm_token_type_ids") + if mm_token_type_ids is None: + mm_token_type_ids = torch.zeros_like(input_ids, dtype=torch.int32) + mm_token_type_ids[input_ids == self.config.image_token_id] = 1 + mm_token_type_ids[input_ids == self.config.video_token_id] = 2 + position_ids, rope_deltas = self.get_rope_index( + input_ids, + mm_token_type_ids, + image_grid_thw, + video_grid_thw, + attention_mask=attention_mask_tensor, + ) + else: + position_ids, rope_deltas = self.get_rope_index( + input_ids, + image_grid_thw, + video_grid_thw, + attention_mask=attention_mask_tensor, + ) self.rope_deltas = rope_deltas # then use the prev pre-calculated rope-deltas to get the correct position ids else: From 98579543c21c54c371030b6ad0d27c7381a7171f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 23 Apr 2026 18:55:00 +0200 Subject: [PATCH 14/87] add torchcodec tests install --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f5c42220ac..1d9a43011a 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "open_clip_torch>=2.26.1", "peft", "datasets>=1.4.0", + "torchcodec", "tbb", "langchain-huggingface", "hf_xet", From 95a6efda61a7fd19857735d8ce7c2e2d80ce5538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 11:44:12 +0200 Subject: [PATCH 15/87] add missing get_vision_position_ids --- optimum/intel/openvino/modeling_visual_language.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index a713c465cb..1e7c281348 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -2577,6 +2577,7 @@ class QWen2VLModelOutputWithPast(ModelOutput): class _OVQwen2VLForCausalLM(OVModelForVisualCausalLM): get_rope_index = Qwen2VLModel.get_rope_index + get_vision_position_ids = getattr(Qwen2_5_VLModel, "get_vision_position_ids", None) additional_parts = ["vision_embeddings_merger"] def __init__( From dbc13a96ebd7992d4e29eebaa0a2263ca14ca561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 15:28:26 +0200 Subject: [PATCH 16/87] add transformers v5.3 check --- optimum/intel/openvino/modeling_visual_language.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 1e7c281348..282b7148a5 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -2629,7 +2629,9 @@ def prepare_inputs_for_generation( **kwargs, ): # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 - if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if is_transformers_version(">=", "5.3") and ( + cache_position is None or (not is_first_iteration and cache_position[0] == 0) + ): if next_sequence_length is not None: past_len = input_ids.shape[1] - next_sequence_length cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) @@ -2950,7 +2952,9 @@ def prepare_inputs_for_generation( **kwargs, ): # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 - if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if is_transformers_version(">=", "5.3") and ( + cache_position is None or (not is_first_iteration and cache_position[0] == 0) + ): if next_sequence_length is not None: past_len = input_ids.shape[1] - next_sequence_length cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) @@ -3308,7 +3312,9 @@ def prepare_inputs_for_generation( **kwargs, ): # reconstruct cache_position as partially removed in v5.3 and totally removed in v5.5 - if cache_position is None or (not is_first_iteration and cache_position[0] == 0): + if is_transformers_version(">=", "5.3") and ( + cache_position is None or (not is_first_iteration and cache_position[0] == 0) + ): if next_sequence_length is not None: past_len = input_ids.shape[1] - next_sequence_length cache_position = torch.arange(past_len, past_len + next_sequence_length, device=input_ids.device) From fd94a591c70e0aaaff71f9804da00957573dd42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 15:40:31 +0200 Subject: [PATCH 17/87] workflow --- .github/workflows/test_openvino.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index dc31fefd85..b2588cdba5 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -51,6 +51,10 @@ jobs: with: python-version: "3.10" + - name: Install dependencies + run: | + apt-get update && apt-get install -y ffmpeg + - name: Install dependencies run: | pip install --upgrade pip uv From 43bd8169455009d3dacb54589851173ca816f12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 15:52:18 +0200 Subject: [PATCH 18/87] worflow --- .github/workflows/test_openvino.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index b2588cdba5..ca1a80dd17 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -53,7 +53,7 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install -y ffmpeg + sudo apt-get update && sudo apt-get install -y ffmpeg - name: Install dependencies run: | From 31ffbb4140402af4f0f0003c63a3b6cbcbd24536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 17:03:50 +0200 Subject: [PATCH 19/87] fix for Qwen2_5 --- optimum/intel/openvino/modeling_visual_language.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 282b7148a5..4b1c4346ca 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -3238,6 +3238,16 @@ def preprocess_inputs( text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True) inputs = processor(images=image, text=text_prompt, videos=video, return_tensors="pt") + + # since transformers v5.3, get_rope_index consumes one second_per_grid_ts entry per + # non-text token group (images and videos), but the processor only populates it for videos + if is_transformers_version(">=", "5.3.0") and "second_per_grid_ts" in inputs and "mm_token_type_ids" in inputs: + non_text_groups = inputs["mm_token_type_ids"][0].unique_consecutive() + non_text_groups = non_text_groups[non_text_groups != 0].tolist() + video_ts_iter = iter(inputs["second_per_grid_ts"].tolist()) + full_ts = [next(video_ts_iter, 1.0) if g == 2 else 1.0 for g in non_text_groups] + inputs["second_per_grid_ts"] = torch.tensor(full_ts, dtype=inputs["second_per_grid_ts"].dtype) + return inputs From 8ae40d3814192dfcf7ae3cd3e9b5b89680e01728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 18:26:44 +0200 Subject: [PATCH 20/87] revert --- .github/workflows/test_openvino.yml | 4 ---- setup.py | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index ca1a80dd17..dc31fefd85 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -51,10 +51,6 @@ jobs: with: python-version: "3.10" - - name: Install dependencies - run: | - sudo apt-get update && sudo apt-get install -y ffmpeg - - name: Install dependencies run: | pip install --upgrade pip uv diff --git a/setup.py b/setup.py index 1d9a43011a..12ff9e0391 100644 --- a/setup.py +++ b/setup.py @@ -56,8 +56,7 @@ "sentence-transformers<5.4.0", "open_clip_torch>=2.26.1", "peft", - "datasets>=1.4.0", - "torchcodec", + "datasets>=1.4.0,<4.0.0", "tbb", "langchain-huggingface", "hf_xet", From 6c05f5453fef11be667e0ffde9d10291894d204c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 18:30:01 +0200 Subject: [PATCH 21/87] disable for datasets metric test when datasets =", "5.3"), - reason="requires transformers < v5.3 since question-answering pipeline is deprecated in v5.3", + is_transformers_version(">=", "5.3") or is_datasets_version("<", "4"), + reason="requires datasets >= 4 or transformers < v5.3 since question-answering pipeline is deprecated in v5.3", ) def test_metric(self): model_id = "distilbert-base-cased-distilled-squad" From c09aab19f9ff00a5dc2d0ad01895c93b869ed555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 18:31:12 +0200 Subject: [PATCH 22/87] transformers v5.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12ff9e0391..359b6ecb86 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", - "transformers>=4.57,<5.4", + "transformers>=4.57,<5.5", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From 24982a5fa7f539a24ec2dce1ba17b634662f5e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 24 Apr 2026 21:19:07 +0200 Subject: [PATCH 23/87] add missing --- tests/openvino/test_modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 8c0eb8cf31..71767778db 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -97,7 +97,7 @@ TemporaryDirectory, ) from optimum.intel.pipelines import pipeline as optimum_pipeline -from optimum.intel.utils.import_utils import _langchain_hf_available, is_transformers_version +from optimum.intel.utils.import_utils import _langchain_hf_available, is_datasets_version, is_transformers_version from optimum.intel.utils.modeling_utils import _find_files_matching_pattern from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_2_SUBFOLDER, From a80912ea6aaf3daf40fa6481c93e504e371968c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 27 Apr 2026 18:09:52 +0200 Subject: [PATCH 24/87] use sdpa_mask instead of sdpa_mask_without_vmap for transformers v5 or higher --- optimum/exporters/openvino/model_patcher.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 4ae2203561..858a361135 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -250,13 +250,15 @@ def patch_cos_sin_cached_fp32(model): dtype=torch.float32, ) - # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 # Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min def eager_mask_without_vmap(*args, **kwargs) -> Optional[torch.Tensor]: kwargs.pop("allow_is_causal_skip", None) dtype = kwargs.get("dtype", torch.float32) - mask = sdpa_mask_without_vmap(*args, allow_is_causal_skip=False, **kwargs) + if is_transformers_version(">=", "5"): + mask = sdpa_mask(*args, use_vmap=False, **kwargs) + else: + mask = sdpa_mask_without_vmap(*args, allow_is_causal_skip=False, **kwargs) # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf mask = torch.where( From 1652bc834934df2500480d0eb308ae1fad6e875b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 27 Apr 2026 18:47:06 +0200 Subject: [PATCH 25/87] fix eager mask --- optimum/exporters/openvino/model_patcher.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 858a361135..6b62ee1a7f 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -250,15 +250,21 @@ def patch_cos_sin_cached_fp32(model): dtype=torch.float32, ) + # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 # Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min -def eager_mask_without_vmap(*args, **kwargs) -> Optional[torch.Tensor]: +def eager_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: kwargs.pop("allow_is_causal_skip", None) + kwargs.pop("use_vmap", None) dtype = kwargs.get("dtype", torch.float32) - if is_transformers_version(">=", "5"): - mask = sdpa_mask(*args, use_vmap=False, **kwargs) + if is_transformers_version(">=", "5.4"): + q_length = kwargs.pop("q_length") + if isinstance(q_length, torch.Tensor): + q_offset = kwargs.pop("q_offset", 0) + q_length = torch.arange(q_offset, q_offset + q_length, device=q_length.device) + mask = sdpa_mask(batch_size=batch_size, q_length=q_length, use_vmap=False, **kwargs) else: - mask = sdpa_mask_without_vmap(*args, allow_is_causal_skip=False, **kwargs) + mask = sdpa_mask_without_vmap(batch_size=batch_size, allow_is_causal_skip=False, **kwargs) # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf mask = torch.where( From 2699f223019f994d423f2644c955dd85e89aef4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 28 Apr 2026 11:29:41 +0200 Subject: [PATCH 26/87] ov_sdpa_mask_without_vmap --- optimum/exporters/openvino/model_patcher.py | 36 ++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 6b62ee1a7f..7e6b85186c 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -251,27 +251,33 @@ def patch_cos_sin_cached_fp32(model): ) -# Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 -# Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min -def eager_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: - kwargs.pop("allow_is_causal_skip", None) +def ov_sdpa_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: kwargs.pop("use_vmap", None) - dtype = kwargs.get("dtype", torch.float32) if is_transformers_version(">=", "5.4"): - q_length = kwargs.pop("q_length") + q_length = kwargs.pop("q_length", None) if isinstance(q_length, torch.Tensor): q_offset = kwargs.pop("q_offset", 0) q_length = torch.arange(q_offset, q_offset + q_length, device=q_length.device) - mask = sdpa_mask(batch_size=batch_size, q_length=q_length, use_vmap=False, **kwargs) + return sdpa_mask(batch_size=batch_size, q_length=q_length, use_vmap=False, **kwargs) else: - mask = sdpa_mask_without_vmap(batch_size=batch_size, allow_is_causal_skip=False, **kwargs) - # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not - # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf - mask = torch.where( - mask, - torch.tensor(0.0, device=mask.device, dtype=dtype), - torch.tensor(torch.finfo(torch.float16).min, device=mask.device, dtype=dtype), - ) + return sdpa_mask_without_vmap(batch_size=batch_size, **kwargs) + + +# Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 +# Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min +def eager_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: + kwargs.pop("allow_is_causal_skip", None) + kwargs.pop("allow_torch_fix", None) + dtype = kwargs.pop("dtype", torch.float32) + mask = ov_sdpa_mask_without_vmap(batch_size, allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) + if mask is not None: + # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not + # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf + mask = torch.where( + mask, + torch.tensor(0.0, device=mask.device, dtype=dtype), + torch.tensor(torch.finfo(torch.float16).min, device=mask.device, dtype=dtype), + ) return mask From d1a61da0f59ed34ab605f3b201903cd984a4f866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 28 Apr 2026 15:44:38 +0200 Subject: [PATCH 27/87] remove inheritance to Qwen3VLVisionModel and Qwen3VLModel --- optimum/exporters/openvino/model_patcher.py | 10 +++++----- optimum/intel/openvino/modeling_visual_language.py | 14 ++++++-------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 7e6b85186c..20a42fc096 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -251,25 +251,25 @@ def patch_cos_sin_cached_fp32(model): ) -def ov_sdpa_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: +def ov_sdpa_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: kwargs.pop("use_vmap", None) if is_transformers_version(">=", "5.4"): q_length = kwargs.pop("q_length", None) if isinstance(q_length, torch.Tensor): q_offset = kwargs.pop("q_offset", 0) q_length = torch.arange(q_offset, q_offset + q_length, device=q_length.device) - return sdpa_mask(batch_size=batch_size, q_length=q_length, use_vmap=False, **kwargs) + return sdpa_mask(q_length=q_length, use_vmap=False, **kwargs) else: - return sdpa_mask_without_vmap(batch_size=batch_size, **kwargs) + return sdpa_mask_without_vmap(**kwargs) # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 # Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min -def eager_mask_without_vmap(batch_size, **kwargs) -> Optional[torch.Tensor]: +def eager_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: kwargs.pop("allow_is_causal_skip", None) kwargs.pop("allow_torch_fix", None) dtype = kwargs.pop("dtype", torch.float32) - mask = ov_sdpa_mask_without_vmap(batch_size, allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) + mask = ov_sdpa_mask_without_vmap(allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) if mask is not None: # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 4b1c4346ca..1714c89b6b 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -3251,14 +3251,12 @@ def preprocess_inputs( return inputs -# The inheritance from Qwen3VLModel is needed to get access to methods: -# get_placeholder_mask(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L1066 -# get_rope_index(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L916 -# get_video_features(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L1035 -# -# and inheritance from Qwen3VLVisionModel is needed for accessing the following method: -# rot_pos_emb(): https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py#L603 -class _OVQwen3VLForCausalLM(OVModelForVisualCausalLM, Qwen3VLModel, Qwen3VLVisionModel): +class _OVQwen3VLForCausalLM(OVModelForVisualCausalLM): + get_placeholder_mask = Qwen3VLModel.get_placeholder_mask + get_rope_index = Qwen3VLModel.get_rope_index + get_video_features = Qwen3VLModel.get_video_features + rot_pos_emb = Qwen3VLVisionModel.rot_pos_emb + get_vision_position_ids = getattr(Qwen3VLModel, "get_vision_position_ids", None) additional_parts = ["vision_embeddings_merger", "vision_embeddings_pos"] def __init__( From dbdf3afa9e1fd2b2955b97be603b56c943d0e840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 28 Apr 2026 15:47:14 +0200 Subject: [PATCH 28/87] transformers v5.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 359b6ecb86..12ff9e0391 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", - "transformers>=4.57,<5.5", + "transformers>=4.57,<5.4", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From fd8d155500df01626d3b7e278fc2cc84b70b8ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 28 Apr 2026 16:32:37 +0200 Subject: [PATCH 29/87] transformers-v5 branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12ff9e0391..96a8757a9c 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", - "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@xadupre/transformers5", + "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@transformers-v5", "transformers>=4.57,<5.4", "setuptools", "huggingface-hub>=0.23.2,<2.0", From 888bfb9d8b8bc7284437a44a1cd6f24a30582109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 28 Apr 2026 19:46:17 +0200 Subject: [PATCH 30/87] transformers v5.4 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index be325e7d33..6b89656861 100644 --- a/setup.py +++ b/setup.py @@ -28,8 +28,8 @@ INSTALL_REQUIRE = [ "torch>=2.1", - "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@transformers-v5", - "transformers>=4.57,<5.4", + "optimum-onnx@git+https://github.com/huggingface/optimum-onnx.git@transformers-v5.5", + "transformers>=4.57,<5.5", "setuptools", "huggingface-hub>=0.23.2,<2.0", "nncf>=2.19.0", From 60582f8e7dcbe3752bff789042a3009f38cf51ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 17:01:52 +0200 Subject: [PATCH 31/87] add transformers v5.3 for tests temporarily --- .github/workflows/test_openvino.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index dc31fefd85..48f33bc69c 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -38,7 +38,7 @@ jobs: "*diffusion*", "*quantization*", ] - transformers-version: ["4.57.6", "latest"] + transformers-version: ["4.57.6", "5.3.0", "latest"] runs-on: ubuntu-22.04 From ad5aea60c1cf493cada28c8fbf35c02ef8c9b7c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 17:03:13 +0200 Subject: [PATCH 32/87] use sdpa_mask for transformers v5 or higher --- optimum/exporters/openvino/model_patcher.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index e0d5edc775..7c02bc5ded 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -52,8 +52,8 @@ ModelPatcher, gpt_oss_forward, override_arguments, - sdpa_mask_without_vmap, ) +from optimum.exporters.onnx.model_patcher import sdpa_mask_without_vmap as sdpa_mask_without_vmap_legacy from optimum.intel.utils.import_utils import ( is_diffusers_version, is_openvino_version, @@ -251,16 +251,17 @@ def patch_cos_sin_cached_fp32(model): ) -def ov_sdpa_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: +def sdpa_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: kwargs.pop("use_vmap", None) - if is_transformers_version(">=", "5.4"): + if is_transformers_version("<", "5"): + return sdpa_mask_without_vmap_legacy(**kwargs) + elif is_transformers_version(">=", "5.4") and is_transformers_version("<=", "5.7"): q_length = kwargs.pop("q_length", None) - if isinstance(q_length, torch.Tensor): - q_offset = kwargs.pop("q_offset", 0) - q_length = torch.arange(q_offset, q_offset + q_length, device=q_length.device) - return sdpa_mask(q_length=q_length, use_vmap=False, **kwargs) + q_offset = kwargs.pop("q_offset", 0) + cache_position = torch.arange(q_offset, q_offset + q_length, device=q_length.device) + return sdpa_mask(q_length=cache_position, use_vmap=False, **kwargs) else: - return sdpa_mask_without_vmap(**kwargs) + return sdpa_mask(use_vmap=False, **kwargs) # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 @@ -269,7 +270,7 @@ def eager_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: kwargs.pop("allow_is_causal_skip", None) kwargs.pop("allow_torch_fix", None) dtype = kwargs.pop("dtype", torch.float32) - mask = ov_sdpa_mask_without_vmap(allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) + mask = sdpa_mask_without_vmap(allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) if mask is not None: # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf From f9788e32f44590969e58368a1715246119c5005d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 18:15:54 +0200 Subject: [PATCH 33/87] remove tests for v5.4 for incompatible models --- tests/openvino/test_decoder.py | 9 ++++++--- tests/openvino/test_seq2seq.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/openvino/test_decoder.py b/tests/openvino/test_decoder.py index ba45d393f5..4e58e27478 100644 --- a/tests/openvino/test_decoder.py +++ b/tests/openvino/test_decoder.py @@ -104,20 +104,23 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES += SUPPORTED_SSM_ARCHITECTURES - if is_transformers_version(">=", "4.48.0"): + if is_transformers_version(">=", "4.48.0") and is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("cohere2",) if is_transformers_version(">=", "4.46.0"): - SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "phimoe") + SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo") if is_transformers_version("<", "4.54.0"): SUPPORTED_ARCHITECTURES += ("deepseek",) + if is_transformers_version("!=", "5.4"): + SUPPORTED_ARCHITECTURES += ("phimoe",) + # gptq and awq install disabled for windows test environment if platform.system() != "Windows" and is_transformers_version("<", "4.56.0"): SUPPORTED_ARCHITECTURES += ("opt_gptq", "mixtral_awq") - if is_transformers_version(">", "4.47"): + if is_transformers_version(">", "4.47") and is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("olmo2",) if is_transformers_version(">=", "4.50"): diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 4a4affeec7..00efef968c 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -143,8 +143,6 @@ class OVModelForSeq2SeqLMIntegrationTest(OVSeq2SeqTestMixin): "blenderbot", "blenderbot-small", "longt5", - "m2m_100", - "mbart", "pegasus", "t5", ) @@ -154,6 +152,12 @@ class OVModelForSeq2SeqLMIntegrationTest(OVSeq2SeqTestMixin): GENERATION_LENGTH = 100 SPEEDUP_CACHE = 1.1 UNSUPPORTED_ARCHITECTURES = set() + + if is_transformers_version("!=", "5.4"): + SUPPORTED_ARCHITECTURES += ("m2m_100", "mbart") + else: + UNSUPPORTED_ARCHITECTURES.update({"m2m_100", "mbart"}) + if not (is_openvino_version(">=", "2025.3.0") and is_openvino_version("<", "2026.1")) and is_transformers_version( "<", "5" ): @@ -445,10 +449,17 @@ def test_pipeline(self, model_arch): class OVModelForVision2SeqIntegrationTest(OVSeq2SeqTestMixin): - SUPPORTED_ARCHITECTURES = ["vision-encoder-decoder", "trocr", "donut"] + SUPPORTED_ARCHITECTURES = ["vision-encoder-decoder", "trocr"] # GOT-OCR2 models shouldn't be exported using the task image-to-text (currently equivalent to exporting the model using image-text-to-text) and will be deprecated v1.29 # TODO: move pix2struct tests from OVModelForPix2StructIntegrationTest + UNSUPPORTED_ARCHITECTURES = {"got_ocr2", "pix2struct"} + + if is_transformers_version("!=", "5.4"): + SUPPORTED_ARCHITECTURES += ("donut",) + else: + UNSUPPORTED_ARCHITECTURES.add("donut") + TASK = "image-to-text" OVMODEL_CLASS = OVModelForVision2Seq AUTOMODEL_CLASS = transformers_auto_class From ea462e4822729f5c73179cd5b51111c1a2a2af69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 18:23:14 +0200 Subject: [PATCH 34/87] add comments --- tests/openvino/test_decoder.py | 3 +++ tests/openvino/test_seq2seq.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tests/openvino/test_decoder.py b/tests/openvino/test_decoder.py index 4e58e27478..06a0243048 100644 --- a/tests/openvino/test_decoder.py +++ b/tests/openvino/test_decoder.py @@ -104,6 +104,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): SUPPORTED_ARCHITECTURES += SUPPORTED_SSM_ARCHITECTURES + # config loading failing coming from type mismatch coming from transformers v5.4 if is_transformers_version(">=", "4.48.0") and is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("cohere2",) @@ -113,6 +114,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): if is_transformers_version("<", "4.54.0"): SUPPORTED_ARCHITECTURES += ("deepseek",) + # config loading failing coming from type mismatch coming from transformers v5.4 if is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("phimoe",) @@ -120,6 +122,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): if platform.system() != "Windows" and is_transformers_version("<", "4.56.0"): SUPPORTED_ARCHITECTURES += ("opt_gptq", "mixtral_awq") + # config loading failing coming from type mismatch coming from transformers v5.4 if is_transformers_version(">", "4.47") and is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("olmo2",) diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 00efef968c..2028c572dc 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -153,6 +153,7 @@ class OVModelForSeq2SeqLMIntegrationTest(OVSeq2SeqTestMixin): SPEEDUP_CACHE = 1.1 UNSUPPORTED_ARCHITECTURES = set() + # config loading failing coming from type mismatch coming from transformers v5.4 if is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("m2m_100", "mbart") else: @@ -455,6 +456,7 @@ class OVModelForVision2SeqIntegrationTest(OVSeq2SeqTestMixin): UNSUPPORTED_ARCHITECTURES = {"got_ocr2", "pix2struct"} + # config loading failing coming from type mismatch coming from transformers v5.4 if is_transformers_version("!=", "5.4"): SUPPORTED_ARCHITECTURES += ("donut",) else: From 9acdc498366d45a1d4f843429d17604c559aaa78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 18:34:24 +0200 Subject: [PATCH 35/87] simplify --- tests/openvino/test_seq2seq.py | 71 ++++++++++++---------------------- 1 file changed, 25 insertions(+), 46 deletions(-) diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 2028c572dc..84f10f94c5 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -151,27 +151,18 @@ class OVModelForSeq2SeqLMIntegrationTest(OVSeq2SeqTestMixin): TASK = "text2text-generation" GENERATION_LENGTH = 100 SPEEDUP_CACHE = 1.1 - UNSUPPORTED_ARCHITECTURES = set() - # config loading failing coming from type mismatch coming from transformers v5.4 - if is_transformers_version("!=", "5.4"): - SUPPORTED_ARCHITECTURES += ("m2m_100", "mbart") - else: - UNSUPPORTED_ARCHITECTURES.update({"m2m_100", "mbart"}) - - if not (is_openvino_version(">=", "2025.3.0") and is_openvino_version("<", "2026.1")) and is_transformers_version( - "<", "5" - ): - # There are known issues with marian model on OpenVINO 2025.3.x and 2025.4.x - SUPPORTED_ARCHITECTURES += ("marian",) - else: - UNSUPPORTED_ARCHITECTURES.add("marian") - - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version("<", "5"): - SUPPORTED_ARCHITECTURES += ("mt5",) - else: - UNSUPPORTED_ARCHITECTURES.add("mt5") + # known issues with marian on OpenVINO 2025.3.x and 2025.4.x + # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly (mt5) + _is_model_supported = { + "m2m_100": is_transformers_version("!=", "5.4"), + "mbart": is_transformers_version("!=", "5.4"), + "marian": not (is_openvino_version(">=", "2025.3.0") and is_openvino_version("<", "2026.1")) + and is_transformers_version("<", "5"), + "mt5": is_transformers_version("<", "5"), + } + SUPPORTED_ARCHITECTURES += tuple(arch for arch, supported in _is_model_supported.items() if supported) + UNSUPPORTED_ARCHITECTURES = {arch for arch, supported in _is_model_supported.items() if not supported} SUPPORT_STATEFUL = ("t5", "mt5", "longt5") if is_transformers_version(">=", "4.52.0"): @@ -454,13 +445,12 @@ class OVModelForVision2SeqIntegrationTest(OVSeq2SeqTestMixin): # GOT-OCR2 models shouldn't be exported using the task image-to-text (currently equivalent to exporting the model using image-text-to-text) and will be deprecated v1.29 # TODO: move pix2struct tests from OVModelForPix2StructIntegrationTest - UNSUPPORTED_ARCHITECTURES = {"got_ocr2", "pix2struct"} - # config loading failing coming from type mismatch coming from transformers v5.4 - if is_transformers_version("!=", "5.4"): - SUPPORTED_ARCHITECTURES += ("donut",) - else: - UNSUPPORTED_ARCHITECTURES.add("donut") + _is_model_supported = {"donut": is_transformers_version("!=", "5.4")} + SUPPORTED_ARCHITECTURES += [arch for arch, supported in _is_model_supported.items() if supported] + UNSUPPORTED_ARCHITECTURES = {"got_ocr2", "pix2struct"} | { + arch for arch, supported in _is_model_supported.items() if not supported + } TASK = "image-to-text" OVMODEL_CLASS = OVModelForVision2Seq @@ -581,18 +571,10 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): if is_transformers_version(">=", "4.46.0"): SUPPORTED_ARCHITECTURES += ["maira2"] - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version("<", "5"): - SUPPORTED_ARCHITECTURES += ["idefics3"] - if is_transformers_version(">=", "4.49.0"): SUPPORTED_ARCHITECTURES += ["qwen2_5_vl"] SUPPORT_VIDEO.append("qwen2_5_vl") - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version("<", "5"): - SUPPORTED_ARCHITECTURES += ["got_ocr2"] - if is_transformers_version("<", "4.54.0"): # remote code models differs after transformers v4.54 SUPPORTED_ARCHITECTURES += ["phi4mm"] @@ -600,14 +582,6 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): if is_transformers_version(">=", "4.50"): SUPPORTED_ARCHITECTURES += ["gemma3"] - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version("<", "5"): - SUPPORTED_ARCHITECTURES += ["smolvlm"] - - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version(">=", "4.51") and is_transformers_version("<", "5"): - # SUPPORTED_ARCHITECTURES += ["llama4", "phi4_multimodal"] - SUPPORTED_ARCHITECTURES += ["llama4"] if is_transformers_version("<", "4.52"): SUPPORTED_ARCHITECTURES += ["minicpmo"] @@ -624,10 +598,15 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): SUPPORTED_ARCHITECTURES += ["internvl_chat", "minicpmv"] # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly - if is_transformers_version("<", "5"): - SUPPORTED_ARCHITECTURES += ("llava_next_video",) - else: - UNSUPPORTED_ARCHITECTURES.update({"got_ocr2", "idefics3", "llama4", "llava_next_video", "smolvlm"}) + _is_model_supported = { + "idefics3": is_transformers_version(">=", "4.46.0") and is_transformers_version("<", "5"), + "got_ocr2": is_transformers_version(">=", "4.49.0") and is_transformers_version("<", "5"), + "smolvlm": is_transformers_version(">=", "4.50") and is_transformers_version("<", "5"), + "llama4": is_transformers_version(">=", "4.51") and is_transformers_version("<", "5"), + "llava_next_video": is_transformers_version("<", "5"), + } + SUPPORTED_ARCHITECTURES += [arch for arch, supported in _is_model_supported.items() if supported] + UNSUPPORTED_ARCHITECTURES.update(arch for arch, supported in _is_model_supported.items() if not supported) REMOTE_CODE_MODELS = ["internvl_chat", "minicpmv", "minicpmo", "llava-qwen2", "phi3_v", "maira2", "phi4mm"] IMAGE = Image.open( requests.get( From 588f89e0b8fc4fcbb188c1748adf4dcac7274583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 29 Apr 2026 18:52:15 +0200 Subject: [PATCH 36/87] add comments --- tests/openvino/test_seq2seq.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index 84f10f94c5..49a03539b8 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -151,14 +151,14 @@ class OVModelForSeq2SeqLMIntegrationTest(OVSeq2SeqTestMixin): TASK = "text2text-generation" GENERATION_LENGTH = 100 SPEEDUP_CACHE = 1.1 - # config loading failing coming from type mismatch coming from transformers v5.4 - # known issues with marian on OpenVINO 2025.3.x and 2025.4.x - # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly (mt5) _is_model_supported = { + # config loading failing coming from type mismatch coming from transformers v5.4 "m2m_100": is_transformers_version("!=", "5.4"), "mbart": is_transformers_version("!=", "5.4"), + # known issues with marian on OpenVINO 2025.3.x and 2025.4.x "marian": not (is_openvino_version(">=", "2025.3.0") and is_openvino_version("<", "2026.1")) and is_transformers_version("<", "5"), + # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly (mt5) "mt5": is_transformers_version("<", "5"), } SUPPORTED_ARCHITECTURES += tuple(arch for arch, supported in _is_model_supported.items() if supported) From 8b40b7a098d6d3a86f3cb6808fd6e0411fe5f7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 18 May 2026 16:55:45 +0200 Subject: [PATCH 37/87] Fix Qwen3_5 model --- .../openvino/modeling_visual_language.py | 70 ++----------------- tests/openvino/test_seq2seq.py | 11 ++- 2 files changed, 17 insertions(+), 64 deletions(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index cdafd3cf72..cd099a49f7 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -32,6 +32,7 @@ from transformers.modeling_outputs import BaseModelOutputWithPooling from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel, VisionRotaryEmbedding +from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5Model, Qwen3_5VisionModel from transformers.models.qwen3_vl.modeling_qwen3_vl import ( Qwen3VLModel, Qwen3VLVisionModel, @@ -3268,63 +3269,6 @@ def preprocess_inputs( return inputs - -if is_transformers_version(">=", "4.57.0"): - from transformers.models.qwen3_vl.modeling_qwen3_vl import ( - Qwen3VLModel, - Qwen3VLVisionModel, - Qwen3VLVisionRotaryEmbedding, - ) -else: - - class Qwen3VLModel: - pass - - class Qwen3VLVisionModel: - pass - - class Qwen3_5Model: - pass - - class Qwen3_5VisionModel: - pass - - -if is_transformers_version(">=", "5.2.0"): - from transformers.models.qwen3_5.modeling_qwen3_5 import ( - Qwen3_5Model, - Qwen3_5VisionModel, - Qwen3_5VisionRotaryEmbedding, - ) -else: - - class Qwen3_5Model: - pass - - class Qwen3_5VisionModel: - pass - - class Qwen3_5VisionRotaryEmbedding: - pass - - -if is_transformers_version(">=", "5.2.0"): - from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import ( - Qwen3_5MoeModel, - Qwen3_5MoeVisionModel, - ) -else: - - class Qwen3_5MoeModel: - pass - - class Qwen3_5MoeVisionModel: - pass - - - - - class _OVQwen3VLForCausalLM(OVModelForVisualCausalLM): get_placeholder_mask = Qwen3VLModel.get_placeholder_mask get_rope_index = Qwen3VLModel.get_rope_index @@ -5289,12 +5233,12 @@ def prepare_inputs_for_generation( return model_inputs -# The inheritance from Qwen3_5Model is needed to get access to methods: -# get_placeholder_mask(), get_rope_index(), get_image_features(), get_video_features(), compute_3d_position_ids() -# -# and inheritance from Qwen3_5VisionModel is needed for accessing the following method: -# rot_pos_emb() -class _OVQwen3_5ForCausalLM(OVModelForVisualCausalLM, Qwen3_5Model, Qwen3_5VisionModel): +class _OVQwen3_5ForCausalLM(OVModelForVisualCausalLM): + get_placeholder_mask = Qwen3_5Model.get_placeholder_mask + get_rope_index = Qwen3_5Model.get_rope_index + get_image_features = Qwen3_5Model.get_image_features + get_video_features = Qwen3_5Model.get_video_features + rot_pos_emb = Qwen3_5VisionModel.rot_pos_emb additional_parts = ["vision_embeddings_merger", "vision_embeddings_pos"] def __init__( diff --git a/tests/openvino/test_seq2seq.py b/tests/openvino/test_seq2seq.py index cd17493484..3bf65b65bd 100644 --- a/tests/openvino/test_seq2seq.py +++ b/tests/openvino/test_seq2seq.py @@ -714,7 +714,16 @@ class OVModelForVisualCausalLMIntegrationTest(OVSeq2SeqTestMixin): } SUPPORTED_ARCHITECTURES += [arch for arch, supported in _is_model_supported.items() if supported] UNSUPPORTED_ARCHITECTURES.update(arch for arch, supported in _is_model_supported.items() if not supported) - REMOTE_CODE_MODELS = ["internvl_chat", "minicpmv", "minicpmo", "llava-qwen2", "phi3_v", "maira2", "phi4mm", "videochat_flash_qwen"] + REMOTE_CODE_MODELS = [ + "internvl_chat", + "minicpmv", + "minicpmo", + "llava-qwen2", + "phi3_v", + "maira2", + "phi4mm", + "videochat_flash_qwen", + ] IMAGE = Image.open( requests.get( TEST_IMAGE_URL, From a95ba3c1d47baf5b8fa7112af9460eabd32031ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 18 May 2026 17:58:30 +0200 Subject: [PATCH 38/87] fix import --- optimum/intel/openvino/modeling_visual_language.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index cd099a49f7..f15d042773 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -32,7 +32,11 @@ from transformers.modeling_outputs import BaseModelOutputWithPooling from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel, VisionRotaryEmbedding -from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5Model, Qwen3_5VisionModel +from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5Model, + Qwen3_5VisionModel, + Qwen3_5VisionRotaryEmbedding, +) from transformers.models.qwen3_vl.modeling_qwen3_vl import ( Qwen3VLModel, Qwen3VLVisionModel, From 1923870ba86ff5ac5443e999bdbcd3e16ae5ef76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Mon, 18 May 2026 18:36:48 +0200 Subject: [PATCH 39/87] fix untested architectures test --- tests/openvino/test_decoder.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/openvino/test_decoder.py b/tests/openvino/test_decoder.py index 06a0243048..708f58d5de 100644 --- a/tests/openvino/test_decoder.py +++ b/tests/openvino/test_decoder.py @@ -27,6 +27,8 @@ DeepseekOpenVINOConfig, LFM2MoeOpenVINOConfig, LFM2OpenVINOConfig, + Qwen3_5MoeTextOpenVINOConfig, + Qwen3_5TextOpenVINOConfig, Qwen3VLOpenVINOConfig, ) from optimum.exporters.openvino.model_patcher import patch_update_causal_mask @@ -324,6 +326,12 @@ def test_find_untested_architectures(self): # qwen3_vl_text a part of qwen3_vl architecture and is tested in seq2seq group if is_transformers_version(">=", str(Qwen3VLOpenVINOConfig.MIN_TRANSFORMERS_VERSION)): supported_architectures -= {"qwen3_vl_text"} + # qwen3_5_text a part of qwen3_5 architecture and is tested in seq2seq group + if is_transformers_version(">=", str(Qwen3_5TextOpenVINOConfig.MIN_TRANSFORMERS_VERSION)): + supported_architectures -= {"qwen3_5_text"} + # qwen3_5_moe_text a part of qwen3_5_moe architecture and is tested in seq2seq group + if is_transformers_version(">=", str(Qwen3_5MoeTextOpenVINOConfig.MIN_TRANSFORMERS_VERSION)): + supported_architectures -= {"qwen3_5_moe_text"} # TODO: add fix for v5 and update MAX_TRANSFORMERS_VERSION accordingly if is_transformers_version(">=", "5"): From ab0132cdf087b7f9c3b65bb0f085161eb49fe4e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 19 May 2026 16:54:04 +0200 Subject: [PATCH 40/87] fix sdpa_mask_without_vmap --- optimum/exporters/openvino/model_patcher.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index db05e45ce8..936df79d5c 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -255,7 +255,11 @@ def sdpa_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: kwargs.pop("use_vmap", None) if is_transformers_version("<", "5"): return sdpa_mask_without_vmap_legacy(**kwargs) - elif is_transformers_version(">=", "5.4") and is_transformers_version("<=", "5.7"): + elif ( + is_transformers_version(">=", "5.4") + and is_transformers_version("<", "5.9") + and isinstance(kwargs.get("q_length", None), torch.Tensor) + ): q_length = kwargs.pop("q_length", None) q_offset = kwargs.pop("q_offset", 0) cache_position = torch.arange(q_offset, q_offset + q_length, device=q_length.device) From 8a34b9541179d9c08d319eac4ec891f01a4b27b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 19 May 2026 17:35:04 +0200 Subject: [PATCH 41/87] fix imports --- .../openvino/modeling_visual_language.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index f15d042773..d58fdb985e 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -32,11 +32,6 @@ from transformers.modeling_outputs import BaseModelOutputWithPooling from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel, VisionRotaryEmbedding -from transformers.models.qwen3_5.modeling_qwen3_5 import ( - Qwen3_5Model, - Qwen3_5VisionModel, - Qwen3_5VisionRotaryEmbedding, -) from transformers.models.qwen3_vl.modeling_qwen3_vl import ( Qwen3VLModel, Qwen3VLVisionModel, @@ -60,6 +55,14 @@ from optimum.intel.utils.import_utils import is_transformers_version +if is_transformers_version(">=", "5.2"): + from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5Model, + Qwen3_5VisionModel, + Qwen3_5VisionRotaryEmbedding, + ) + + if is_transformers_version(">=", "4.46.0"): from transformers import AutoModelForImageTextToText @@ -5238,11 +5241,6 @@ def prepare_inputs_for_generation( class _OVQwen3_5ForCausalLM(OVModelForVisualCausalLM): - get_placeholder_mask = Qwen3_5Model.get_placeholder_mask - get_rope_index = Qwen3_5Model.get_rope_index - get_image_features = Qwen3_5Model.get_image_features - get_video_features = Qwen3_5Model.get_video_features - rot_pos_emb = Qwen3_5VisionModel.rot_pos_emb additional_parts = ["vision_embeddings_merger", "vision_embeddings_pos"] def __init__( @@ -5627,6 +5625,12 @@ def generate(self, *args, **kwargs): return super().generate(*args, **kwargs) +if is_transformers_version(">=", "5.2"): + _OVQwen3_5ForCausalLM.get_placeholder_mask = Qwen3_5Model.get_placeholder_mask + _OVQwen3_5ForCausalLM.get_rope_index = Qwen3_5Model.get_rope_index + _OVQwen3_5ForCausalLM.rot_pos_emb = Qwen3_5VisionModel.rot_pos_emb + + MODEL_TYPE_TO_CLS_MAPPING = { "llava": _OVLlavaForCausalLM, "llava_next": _OVLlavaNextForCausalLM, From e5f0864099fdddc5946cc76db1dfed3feb053b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 2 Jun 2026 17:47:42 +0200 Subject: [PATCH 42/87] add _orig_sdpa_mask_without_vmap --- optimum/exporters/openvino/patching_utils.py | 46 +++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/optimum/exporters/openvino/patching_utils.py b/optimum/exporters/openvino/patching_utils.py index 9c077dc907..d0350c7c50 100644 --- a/optimum/exporters/openvino/patching_utils.py +++ b/optimum/exporters/openvino/patching_utils.py @@ -164,13 +164,57 @@ def find_packed_sequence_indices_patched(position_ids: torch.Tensor) -> torch.Te _prepare_padding_mask_slice = False +# Custom vectorized implementation of sdpa_mask without using vmap +def _orig_sdpa_mask_without_vmap( + batch_size: int, + cache_position: torch.Tensor, + kv_length: int, + kv_offset: int = 0, + mask_function: Callable | None = None, + attention_mask: torch.Tensor | None = None, + local_size: int | None = None, + allow_is_causal_skip: bool = True, + **kwargs, +) -> torch.Tensor | None: + if mask_function is None: + mask_function = causal_mask_function + + q_length = cache_position.shape[0] + # Potentially pad the 2D mask, and slice it correctly + if _prepare_padding_mask_slice: + padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset, _slice=False) + else: + padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset) + + # Under specific conditions, we can avoid materializing the mask, instead relying on the `is_causal` argument + if allow_is_causal_skip and _ignore_causal_mask_sdpa(padding_mask, q_length, kv_length, kv_offset, local_size): + return None + + # Potentially add the padding 2D mask + if padding_mask is not None: + mask_function = and_masks(mask_function, padding_mask_function(padding_mask)) + + # Create broadcatable indices + device = cache_position.device + q_indices = cache_position[None, None, :, None] + head_indices = torch.arange(1, dtype=torch.long, device=device)[None, :, None, None] + batch_indices = torch.arange(batch_size, dtype=torch.long, device=device)[:, None, None, None] + kv_indices = torch.arange(kv_length, dtype=torch.long, device=device)[None, None, None, :] + kv_offset + # Apply mask function element-wise through broadcasting + causal_mask = mask_function(batch_indices, head_indices, q_indices, kv_indices) + # Expand the mask to match batch size and query length if they weren't used in the mask function + causal_mask = causal_mask.expand(batch_size, -1, q_length, kv_length) + + return causal_mask + + # Compatibility wrapper for sdpa_mask_without_vmap from optimum. # The installed optimum version expects (batch_size, cache_position: Tensor, kv_length, ...), # but transformers >= 5.5 passes (batch_size, q_length: int, kv_length: int, q_offset: int, ...). def sdpa_mask_without_vmap(**kwargs): kwargs.pop("use_vmap", None) if is_transformers_version("<", "5"): - return sdpa_mask_without_vmap_legacy(**kwargs) + return _orig_sdpa_mask_without_vmap(**kwargs) elif ( is_transformers_version(">=", "5.4") and is_transformers_version("<", "5.9") From 3e670cd6626393b3240d019cf8f9ccc75162d8d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 2 Jun 2026 17:59:42 +0200 Subject: [PATCH 43/87] fix --- optimum/exporters/openvino/patching_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/exporters/openvino/patching_utils.py b/optimum/exporters/openvino/patching_utils.py index d0350c7c50..956b45d0c4 100644 --- a/optimum/exporters/openvino/patching_utils.py +++ b/optimum/exporters/openvino/patching_utils.py @@ -230,7 +230,7 @@ def sdpa_mask_without_vmap(**kwargs): # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 # Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min -def eager_mask_without_vmap(**kwargs) -> Optional[torch.Tensor]: +def eager_mask_without_vmap(**kwargs): kwargs.pop("allow_is_causal_skip", None) kwargs.pop("allow_torch_fix", None) dtype = kwargs.pop("dtype", torch.float32) From 8a09786c9443f2b72f68f3f673c2de733c1bd3d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 2 Jun 2026 18:01:11 +0200 Subject: [PATCH 44/87] fix --- optimum/exporters/openvino/patching_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/optimum/exporters/openvino/patching_utils.py b/optimum/exporters/openvino/patching_utils.py index 956b45d0c4..2c3128b6bc 100644 --- a/optimum/exporters/openvino/patching_utils.py +++ b/optimum/exporters/openvino/patching_utils.py @@ -38,7 +38,11 @@ if is_transformers_version(">=", "4.53"): from transformers.masking_utils import ( ALL_MASK_ATTENTION_FUNCTIONS, + _ignore_causal_mask_sdpa, + and_masks, + causal_mask_function, eager_mask, + padding_mask_function, prepare_padding_mask, sdpa_mask, ) From 774583f8a2c7ef2f84c2b5fc8f36d32f8b80965e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Tue, 2 Jun 2026 18:05:19 +0200 Subject: [PATCH 45/87] config loading failing coming from type mismatch coming from transformers v5.4 --- tests/openvino/test_decoder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/openvino/test_decoder.py b/tests/openvino/test_decoder.py index 708f58d5de..806517b4b9 100644 --- a/tests/openvino/test_decoder.py +++ b/tests/openvino/test_decoder.py @@ -63,7 +63,6 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "mistral", "mixtral", "mpt", - "mbart", "opt", "pegasus", "phi", @@ -74,7 +73,6 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "gpt_neox_japanese", "xglm", "gemma", - "olmo", "stablelm", "starcoder2", "cohere", @@ -88,6 +86,10 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon_mamba") + # config loading failing coming from type mismatch coming from transformers v5.4 + if is_transformers_version("!=", "5.4"): + SUPPORTED_ARCHITECTURES += ("mbart", "olmo") + if is_transformers_version(">=", "4.49") and is_transformers_version("<", "5"): SUPPORTED_SSM_ARCHITECTURES += ("zamba2",) From ff2b5240c198c1c792d45ac55e9631148b6557df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Wed, 3 Jun 2026 18:14:05 +0200 Subject: [PATCH 46/87] update setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e6dddf8bf9..0d2a611566 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ INSTALL_REQUIRE = [ "torch>=2.1", - "optimum@git+https://github.com/huggingface/optimum.git", + "optimum@git+https://github.com/huggingface/optimum@transformers-5.4", "transformers>=4.57,<5.5", "setuptools", "huggingface-hub>=0.23.2,<2.0", From 2bfe3236d87c2061ad965cecc7a8529d51a8c877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 4 Jun 2026 16:45:09 +0200 Subject: [PATCH 47/87] fix --- optimum/exporters/openvino/model_patcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 199bbce903..c22adc5850 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -7859,6 +7859,9 @@ def lfm2_short_conv_forward_patched( # in transformers < v5 attention_mask was never applied in Lfm2ShortConv https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/lfm2/modeling_lfm2.py#L485 # until a fix was added in https://github.com/huggingface/transformers/pull/41790/ if is_transformers_version(">=", "5"): + # since transformers v5.4, Lfm2ShortConv.slow_forward passes attention_mask as 3 positional arg + if attention_mask is None and is_transformers_version(">=", "5.4"): + attention_mask = cache_position dtype = x.dtype is_decoding = torch.tensor(seqlen == 1, dtype=dtype) x = (x * (attention_mask[:, :seqlen, None] * (1 - is_decoding) + is_decoding)).to(dtype) From 45b37446b10a218a225b73bb221fc4c5325aa26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Thu, 4 Jun 2026 17:27:46 +0200 Subject: [PATCH 48/87] fix --- optimum/intel/openvino/modeling_decoder.py | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index f1a2b14fdc..15ed8c94cc 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -1430,12 +1430,11 @@ def _update_model_kwargs_for_generation( self, outputs: ModelOutput, model_kwargs: Dict[str, Any], num_new_tokens: int = 1, **kwargs ) -> Dict[str, Any]: model_kwargs["cache_params"] = outputs.get("cache_params", None) - if ( - model_kwargs.get("use_cache", True) - and "cache_position" in model_kwargs - and model_kwargs["cache_position"] is not None - ): - model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens + if model_kwargs.get("use_cache", True): + if "cache_position" in model_kwargs and model_kwargs["cache_position"] is not None: + model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens + elif model_kwargs.get("cache_params") is not None: + model_kwargs["cache_position"] = torch.tensor([num_new_tokens], dtype=torch.long) if "attention_mask" in model_kwargs: attention_mask = model_kwargs["attention_mask"] @@ -1458,13 +1457,15 @@ def prepare_inputs_for_generation( # Overwitten -- uses `cache_params` as opposed to `past_key_values` if self.use_cache: - # `cache_position` should have been initialized in `generate` if cache_position is None: - raise ValueError( - "`cache_position` should not be None as it should have been initialized in " - "`model.generate`, you are responsible for passing in a valid `cache_position` if " - "you are calling `prepare_inputs_for_generation` directly with `use_cache=True`" - ) + if is_transformers_version("<", "5.4"): + raise ValueError( + "`cache_position` should not be None as it should have been initialized in " + "`model.generate`, you are responsible for passing in a valid `cache_position` if " + "you are calling `prepare_inputs_for_generation` directly with `use_cache=True`" + ) + # infer from cache_params: None means prefill (0), otherwise means decoding stage + cache_position = torch.tensor([0 if cache_params is None else 1], device=input_ids.device, dtype=torch.long) if cache_position[0] > 0: # decoding stage so it takes the last token input_ids = input_ids[:, -1].unsqueeze(-1) From 9beacbbe0dde005d0329aa697d4841f52cff2479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CElla?= Date: Fri, 5 Jun 2026 15:32:01 +0200 Subject: [PATCH 49/87] minimum required transformers version is v4.57 --- optimum/exporters/openvino/__main__.py | 7 +- optimum/exporters/openvino/convert.py | 6 +- .../exporters/openvino/input_generators.py | 60 +- optimum/exporters/openvino/model_configs.py | 137 +- optimum/exporters/openvino/model_patcher.py | 1539 ++--------------- optimum/exporters/openvino/patching_utils.py | 89 +- optimum/intel/openvino/modeling_decoder.py | 4 +- optimum/intel/openvino/modeling_seq2seq.py | 30 +- .../openvino/modeling_visual_language.py | 27 +- optimum/intel/pipelines/pipeline_base.py | 6 +- tests/openvino/test_decoder.py | 154 +- tests/openvino/test_export.py | 44 +- tests/openvino/test_exporters_cli.py | 155 +- tests/openvino/test_genai.py | 63 +- tests/openvino/test_modeling.py | 12 +- tests/openvino/test_quantization.py | 191 +- tests/openvino/test_seq2seq.py | 105 +- 17 files changed, 395 insertions(+), 2234 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index f3d8fed9de..a1ea6caffe 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -24,7 +24,7 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from requests.exceptions import ConnectionError as RequestsConnectionError -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin +from transformers import AutoConfig, AutoTokenizer, Mxfp4Config, PreTrainedTokenizerBase, ProcessorMixin from transformers.utils import is_torch_available from openvino import Core, Type, save_model @@ -53,9 +53,6 @@ ) -if is_transformers_version(">=", "4.55"): - from transformers import Mxfp4Config - FORCE_ATTN_MODEL_CLASSES = {"phi3_v": "eager", "gemma2": "sdpa", "llama4": "sdpa"} if TYPE_CHECKING: @@ -360,7 +357,7 @@ def main_export( loading_kwargs["config"] = update_config_for_eagle3(config) # mxfp4 quantized model will be dequantized to bf16 - if quant_method == "mxfp4" and is_transformers_version(">=", "4.55"): + if quant_method == "mxfp4": dtype = torch.bfloat16 loading_kwargs["quantization_config"] = Mxfp4Config(dequantize=True) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index dbe8e403cd..e98f7ab237 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -641,11 +641,7 @@ def export_from_model( ensure_export_task_support_stateful(task) or ensure_model_type_support_stateful(model_type) ) - if ( - stateful - and is_encoder_decoder - and not getattr(model, "_supports_cache_class", is_transformers_version(">=", "4.54")) - ): + if stateful and is_encoder_decoder and not getattr(model, "_supports_cache_class", True): stateful = False # TODO: support ov_config.py in the model repo if custom_architecture and custom_export_configs is None: diff --git a/optimum/exporters/openvino/input_generators.py b/optimum/exporters/openvino/input_generators.py index da2fc7a7f7..26e01ef72c 100644 --- a/optimum/exporters/openvino/input_generators.py +++ b/optimum/exporters/openvino/input_generators.py @@ -29,7 +29,6 @@ FalconDummyPastKeyValuesGenerator, MistralDummyPastKeyValuesGenerator, NormalizedTextConfig, - is_transformers_version, ) from optimum.utils.input_generators import DTYPE_MAPPER from optimum.utils.normalized_config import NormalizedConfig, NormalizedVisionConfig @@ -41,46 +40,27 @@ def __init__(self, task: str, normalized_config: NormalizedTextConfig, **kwargs) self.multi_query = normalized_config.multi_query def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): - if is_transformers_version("<", "4.54"): - if self.multi_query: - shape = ( - self.batch_size, - self.sequence_length, - self.hidden_size // self.num_attention_heads * 2, - ) - else: - shape = ( - self.batch_size, - self.num_attention_heads, - self.sequence_length, - self.hidden_size // self.num_attention_heads * 2, - ) - pkv = [ - self.random_float_tensor(shape, framework=framework, dtype=float_dtype) for _ in range(self.num_layers) - ] - + if self.multi_query: + shape = ( + self.batch_size, + 1, + self.sequence_length, + self.hidden_size // self.num_attention_heads, + ) else: - if self.multi_query: - shape = ( - self.batch_size, - 1, - self.sequence_length, - self.hidden_size // self.num_attention_heads, - ) - else: - shape = ( - self.batch_size, - self.num_attention_heads, - self.sequence_length, - self.hidden_size // self.num_attention_heads, - ) - pkv = [ - ( - self.random_float_tensor(shape, framework=framework, dtype=float_dtype), - self.random_float_tensor(shape, framework=framework, dtype=float_dtype), - ) - for _ in range(self.num_layers) - ] + shape = ( + self.batch_size, + self.num_attention_heads, + self.sequence_length, + self.hidden_size // self.num_attention_heads, + ) + pkv = [ + ( + self.random_float_tensor(shape, framework=framework, dtype=float_dtype), + self.random_float_tensor(shape, framework=framework, dtype=float_dtype), + ) + for _ in range(self.num_layers) + ] return pkv diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 32ee9ed288..10796082fb 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -88,11 +88,8 @@ ArcticModelPatcher, BaichuanModelPatcher, BigBirdPegasusModelPatcher, - BlenderbotModelPatcher, - BlenderbotSmallModelPatcher, BloomModelPatcher, ChatGLMModelPatcher, - CLIPModelPatcher, CodeGenModelPatcher, CommonImageEmbeddingsModelPatcher, DBRXModelPatcher, @@ -106,7 +103,6 @@ Gemma4LMModelPatcher, GptJModelPatcher, GptNeoModelPatcher, - GptNeoxModelPatcher, GptOssModelPatcher, GraniteMoeHybridModelPatcher, GraniteMoEModelPatcher, @@ -128,7 +124,6 @@ LlavaQwen2ImageEmbeddingsModelPatcher, MairaImageEmbeddingModelPatcher, MambaPatcher, - MarianModelPatcher, MiniCPM3Patcher, MiniCPMModelPatcher, MiniCPMVImageEmbeddingsModelPatcher, @@ -139,8 +134,6 @@ MPTModelPatcher, OVDecoderModelPatcher, OVSeq2SeqModelPatcher, - PegasusModelPatcher, - PersimmonModelPatcher, Phi3ModelPatcher, Phi3VisionImageEmbeddingsPatcher, Phi4MMAudioEncoderPatcher, @@ -278,38 +271,6 @@ def init_model_configs(): "AutoModelForCausalLM", ) - # since transformers v4.46, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.46.0/src/transformers/models/auto/modeling_auto.py#L776 - if is_transformers_version("<", "4.46"): - TasksManager._CUSTOM_CLASSES[("pt", "llava", "image-text-to-text")] = ( - "transformers", - "LlavaForConditionalGeneration", - ) - TasksManager._CUSTOM_CLASSES[("pt", "llava_next", "image-text-to-text")] = ( - "transformers", - "LlavaNextForConditionalGeneration", - ) - TasksManager._CUSTOM_CLASSES[("pt", "qwen2_vl", "image-text-to-text")] = ( - "transformers", - "Qwen2VLForConditionalGeneration", - ) - - # since transformers v4.50, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.50.0/src/transformers/models/auto/modeling_auto.py#L835 - if is_transformers_version("<", "4.50"): - TasksManager._CUSTOM_CLASSES[("pt", "gemma3", "image-text-to-text")] = ( - "transformers", - "Gemma3ForConditionalGeneration", - ) - - # since transformers v4.52, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.52.0/src/transformers/models/auto/modeling_auto.py#L899 - if is_transformers_version("<", "4.52"): - TasksManager._CUSTOM_CLASSES[("pt", "llava_next_video", "image-text-to-text")] = ( - "transformers", - "AutoModelForVision2Seq", - ) - # Qwen3-ASR is loaded via trust_remote_code; register custom classes for task lookup. if is_transformers_version("==", "4.57.6"): TasksManager._CUSTOM_CLASSES[("pt", "qwen3_asr", "automatic-speech-recognition")] = ( @@ -992,13 +953,11 @@ def inputs(self) -> Dict[str, Dict[int, str]]: ) class PersimmonOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = PersimmonModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager("biogpt", *["text-generation", "text-generation-with-past"], library_name="transformers") -class BioGPTOpenVINOConfig( - TextDecoderWithPositionIdsOpenVINOConfig if is_transformers_version(">=", "4.52.0") else TextDecoderOpenVINOConfig -): +class BioGPTOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig _MODEL_PATCHER = OVDecoderModelPatcher @@ -1055,24 +1014,6 @@ class BloomOpenVINOConfig(TextDecoderOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig.with_args(num_layers="n_layer", num_attention_heads="n_head") _MODEL_PATCHER = BloomModelPatcher - def add_past_key_values(self, inputs_or_outputs: Dict[str, Dict[int, str]], direction: str): - if is_transformers_version(">=", "4.44"): - super().add_past_key_values(inputs_or_outputs, direction) - else: - if direction not in ["inputs", "outputs"]: - raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') - - if direction == "inputs": - decoder_sequence_name = "past_sequence_length" - name = "past_key_values" - else: - decoder_sequence_name = "past_sequence_length + sequence_length" - name = "present" - - for i in range(self._normalized_config.num_layers): - inputs_or_outputs[f"{name}.{i}.key"] = {0: "batch_size * num_heads", 2: decoder_sequence_name} - inputs_or_outputs[f"{name}.{i}.value"] = {0: "batch_size * num_heads", 1: decoder_sequence_name} - @register_in_tasks_manager( "cohere", @@ -1209,7 +1150,7 @@ class MistralOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): ) class GPTNeoxOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = GptNeoxModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager( @@ -1218,7 +1159,7 @@ class GPTNeoxOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): class GPTNeoxJapaneseOpenVINOConfig(TextDecoderOpenVINOConfig): # GPTNeoxJapanese does not require position_ids input. NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = GptNeoxModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager( @@ -1312,7 +1253,6 @@ class SiglipNormalizedConfig(CLIPNormalizedConfig): @register_in_tasks_manager("clip", *["zero-shot-image-classification"], library_name="open_clip") class OpenCLIPOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = CLIPNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1360,7 +1300,6 @@ class OpenCLIPTextOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1415,7 +1354,6 @@ def rename_ambiguous_inputs(self, inputs): ) class CLIPOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = CLIPNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1453,7 +1391,6 @@ class CLIPTextOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1483,7 +1420,6 @@ class CLIPTextWithProjectionOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1507,7 +1443,6 @@ def outputs(self) -> Dict[str, Dict[int, str]]: @register_in_tasks_manager("clip_vision_model", *["feature-extraction"], library_name="transformers") class CLIPVisionModelOpenVINOConfig(VisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1596,10 +1531,7 @@ def generate_dummy_inputs(self, framework: str = "pt", **kwargs): ) dummy_inputs["inputs_embeds"] = inputs_embeds if "token_type_ids" in self.inputs: - if is_transformers_version(">=", "4.53"): - token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1] + pask_key_values[0][0].shape[-2]) - else: - token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1]) + token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1] + pask_key_values[0][0].shape[-2]) dummy_inputs["token_type_ids"] = self.orig_export_config.DUMMY_INPUT_GENERATOR_CLASSES[ 0 ].random_int_tensor(token_type_ids_shape, min_value=0, max_value=2) @@ -3372,12 +3304,6 @@ def inputs(self) -> Dict[str, Dict[int, str]]: # TODO: this can be fixed by generating the correct inputs in the input generator common_inputs["encoder_outputs"] = {0: "batch_size", 1: "encoder_sequence_length"} - if self._behavior in {ConfigBehavior.DECODER, ConfigBehavior.MONOLITH}: - if is_transformers_version(">=", "4.43.0") and is_transformers_version("<", "4.46.0"): - # since https://github.com/huggingface/transformers/pull/31166 - if self._behavior is not ConfigBehavior.ENCODER and self.use_past_in_inputs: - common_inputs["cache_position"] = {0: "decoder_sequence_length"} - return common_inputs @property @@ -4074,7 +4000,7 @@ class SmolVLMOpenVINOConfig(Idefics3OpenVINOConfig): library_name="transformers", ) class BlenderbotOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = BlenderbotModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher @register_in_tasks_manager( @@ -4090,7 +4016,7 @@ class BlenderbotOpenVINOConfig(BartOpenVINOConfig): library_name="transformers", ) class BlenderbotSmallOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = BlenderbotSmallModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher @register_in_tasks_manager( @@ -4106,7 +4032,7 @@ class BlenderbotSmallOpenVINOConfig(BartOpenVINOConfig): library_name="transformers", ) class PegasusOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = PegasusModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -4126,7 +4052,7 @@ def __init__(self, *args, **kwargs): library_name="transformers", ) class MarianOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = MarianModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -4647,9 +4573,7 @@ class Olmo2OOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): @register_in_tasks_manager("opt", *[*COMMON_TEXT_GENERATION_TASKS, "text-classification", "question-answering"]) -class OPTOpenVINOConfig( - TextDecoderWithPositionIdsOpenVINOConfig if is_transformers_version(">=", "4.46.0") else TextDecoderOpenVINOConfig -): +class OPTOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig def __init__(self, *args, **kwargs): @@ -4665,37 +4589,6 @@ class GPTBigCodeOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedConfigManager.get_normalized_config_class("gpt_bigcode") DUMMY_PKV_GENERATOR_CLASS = GPTBigCodeDummyPastKeyValuesGenerator - def add_past_key_values(self, inputs_or_outputs: dict, direction: str): - if is_transformers_version(">=", "4.54"): - super().add_past_key_values(inputs_or_outputs, direction) - else: - if direction not in ["inputs", "outputs"]: - raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') - - if direction == "inputs": - decoder_sequence_name = "past_sequence_length" - name = "past_key_values" - else: - decoder_sequence_name = "past_sequence_length + sequence_length" - name = "present" - - if self._normalized_config.multi_query: - decoder_sequence_dim = 1 - else: - decoder_sequence_dim = 2 - - for i in range(self._normalized_config.num_layers): - inputs_or_outputs[f"{name}.{i}.key_value"] = { - 0: "batch_size", - decoder_sequence_dim: decoder_sequence_name, - } - - def flatten_past_key_values(self, flattened_output, name, idx, t): - if is_transformers_version(">=", "4.54"): - super().flatten_past_key_values(flattened_output, name, idx, t) - else: - flattened_output[f"{name}.{idx}.key_value"] = t - class _Pix2StructNormalizedConfig(NormalizedSeq2SeqConfig): ENCODER_NUM_LAYERS = "vision_config.num_hidden_layers" @@ -4726,12 +4619,6 @@ class Pix2StructOpenVINOConfig(OpenVINOSeq2SeqConfigWithPast): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if is_transformers_version("==", "4.46.0"): - logger.warning( - "Found transformers v4.46.0 while trying to export a Pix2Struct model, " - "this specific version of transformers is broken for this model. Please " - "upgrade to v4.46.1 or higher, or downgrade to v4.45.x.", - ) @property def inputs(self): @@ -5271,7 +5158,6 @@ def outputs(self) -> dict: @register_in_tasks_manager("siglip", *["feature-extraction", "zero-shot-image-classification"]) class SiglipOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = SiglipNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> dict: @@ -5329,7 +5215,6 @@ class SiglipTextWithProjectionOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> dict: @@ -5352,8 +5237,6 @@ def outputs(self) -> dict: @register_in_tasks_manager("siglip-text", *["feature-extraction"]) class SiglipTextOpenVINOConfig(SiglipTextWithProjectionOpenVINOConfig): - _MODEL_PATCHER = CLIPModelPatcher - @property def outputs(self) -> dict: common_outputs = { diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index c22adc5850..cf2bbfea57 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -29,6 +29,12 @@ from transformers.cache_utils import Cache, DynamicCache, EncoderDecoderCache from transformers.configuration_utils import PretrainedConfig from transformers.generation import GenerationMixin +from transformers.masking_utils import ( + ALL_MASK_ATTENTION_FUNCTIONS, + create_causal_mask, + eager_mask, + sdpa_mask, +) from transformers.modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, @@ -43,9 +49,10 @@ LlamaRotaryEmbedding, ) from transformers.models.phi3.modeling_phi3 import apply_rotary_pos_emb, repeat_kv +from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock from transformers.models.speecht5.modeling_speecht5 import SpeechT5EncoderWithSpeechPrenet from transformers.processing_utils import Unpack -from transformers.utils import ModelOutput +from transformers.utils import ModelOutput, TransformersKwargs from optimum.exporters.openvino._ov_ops import convert_recurrent_attention_cell from optimum.exporters.openvino.base import OpenVINOConfig @@ -64,28 +71,6 @@ ) -if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - from transformers.models.clip.modeling_clip import CLIPAttention, CLIPSdpaAttention - - -if is_transformers_version(">=", "4.48"): - from transformers.cache_utils import DynamicCache, EncoderDecoderCache -if is_transformers_version(">=", "4.53"): - from transformers.masking_utils import ( - ALL_MASK_ATTENTION_FUNCTIONS, - eager_mask, - sdpa_mask, - ) - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock - - -if is_transformers_version(">=", "4.54"): - from transformers.masking_utils import create_causal_mask - from transformers.utils import TransformersKwargs -else: - TransformersKwargs = object - - if is_transformers_version(">=", "4.56"): import transformers.masking_utils @@ -206,28 +191,6 @@ def patched_forward(input_ids, attention_mask): self.patched_forward = patched_forward -class CLIPModelPatcher(ModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - self.original_sdpa_forward = CLIPSdpaAttention.forward - CLIPSdpaAttention.forward = CLIPAttention.forward - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - CLIPSdpaAttention.forward = self.original_sdpa_forward - - -# This is a traceable version of the original function, -# the original results in a constant integer due to the use of int(expr) -def _get_feat_extract_output_lengths_patched(self, input_lengths: torch.LongTensor): - output_conv1_length = (input_lengths - 127) // 64 + 1 - output_conv2_length = (output_conv1_length - 7) // 3 + 1 - output_conv3_length = (output_conv2_length - 3) // 2 + 1 - return output_conv3_length - - def _get_model_attribute(model, name): target = getattr(model, "model", model) if is_transformers_version(">=", "5") else model return getattr(target, name) @@ -518,19 +481,14 @@ def __enter__(self): if hasattr(self._model, "model"): patch_cos_sin_cached_fp32(self._model.model) - if is_transformers_version("<", "4.53.0") and hasattr(self._model, "_update_causal_mask"): - self._model._update_causal_mask_original = self._model._update_causal_mask - self._model._update_causal_mask = types.MethodType(_update_causal_mask_patched, self._model) - - if is_transformers_version(">=", "4.53.0"): - # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min - # Although I'm not sure this is the right way to handle this, we are basically pretending that -65,504 is -inf - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) + # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min + # Although I'm not sure this is the right way to handle this, we are basically pretending that -65,504 is -inf + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) - # for decoder models, we use eager mask without vmap for sdpa as well - # to avoid a nan output issue in OpenVINO that only happens in case of: - # non-stateful models on cpu and stateful models on npu - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # for decoder models, we use eager mask without vmap for sdpa as well + # to avoid a nan output issue in OpenVINO that only happens in case of: + # non-stateful models on cpu and stateful models on npu + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) if is_transformers_version(">=", "5"): for module in self._model.modules(): @@ -542,13 +500,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53") and hasattr(self._model, "_update_causal_mask_original"): - self._model._update_causal_mask = self._model._update_causal_mask_original - del self._model._update_causal_mask_original - - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) if is_transformers_version(">=", "5"): for module in self._model.modules(): @@ -826,124 +779,9 @@ def create_embed_positions_buffer(rotary_emb, max_position_embeddings: int = Non return create_sinusoidal_positions(max_position_embeddings, dim, base, inv_freq) -# copied from https://github.com/huggingface/transformers/commit/57d7594a79a9f5d835abf2d4d384db0e4818e548 to unblock export with transformers 4.42 -def _mistral_update_causal_mask( - self, - attention_mask: torch.Tensor, - input_tensor: torch.Tensor, - cache_position: torch.Tensor, - past_key_values: "Cache", - use_cache: bool, - output_attentions: bool, -): - from transformers.cache_utils import SlidingWindowCache, StaticCache - from transformers.modeling_attn_mask_utils import AttentionMaskConverter - - # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static - # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. - # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using - # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 - - if self._attn_implementation == "flash_attention_2": - if attention_mask is not None and use_cache: - is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0] - if is_padding_right: - raise ValueError( - "You are attempting to perform batched generation with padding_side='right'" - " this may lead to unexpected behaviour for Flash Attention version of Mistral. Make sure to " - " call `tokenizer.padding_side = 'left'` before tokenizing the input. " - ) - if attention_mask is not None and 0.0 in attention_mask: - return attention_mask - return None - - # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in - # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail - # to infer the attention mask. - - # cache_position must be valid here no matter which cache we use - past_seen_tokens = cache_position[0] if past_key_values is not None else 0 - using_static_cache = isinstance(past_key_values, StaticCache) - using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) - - if ( - self.config._attn_implementation == "sdpa" - and not (using_static_cache or using_sliding_window_cache) - and not output_attentions - ): - if AttentionMaskConverter._ignore_causal_mask_sdpa( - attention_mask, - inputs_embeds=input_tensor, - past_key_values_length=past_seen_tokens, - sliding_window=self.config.sliding_window, - is_training=self.training, - ): - return None - - dtype, device = input_tensor.dtype, input_tensor.device - min_dtype = torch.finfo(torch.float16).min - sequence_length = input_tensor.shape[1] - # SlidingWindowCache - if using_sliding_window_cache: - target_length = max(sequence_length, self.config.sliding_window) - # StaticCache - elif using_static_cache: - target_length = past_key_values.get_max_length() - # DynamicCache or no cache - else: - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else past_seen_tokens + sequence_length + 1 - ) - - if attention_mask is not None and attention_mask.dim() == 4: - # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing - if attention_mask.max() != 0: - raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`") - causal_mask = attention_mask - else: - causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) - exclude_mask = torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) - if self.config.sliding_window is not None: - if not using_sliding_window_cache or sequence_length > self.config.sliding_window: - exclude_mask = exclude_mask.bitwise_or( - torch.arange(target_length, device=device) - <= (cache_position.reshape(-1, 1) - self.config.sliding_window) - ) - causal_mask *= exclude_mask - causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) - if attention_mask is not None: - causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit - if attention_mask.dim() == 2: - mask_length = attention_mask.shape[-1] - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] - padding_mask = padding_mask == 0 - causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( - padding_mask, min_dtype - ) - - if ( - self.config._attn_implementation == "sdpa" - and attention_mask is not None - and attention_mask.device.type == "cuda" - and not output_attentions - ): - # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when - # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. - # Details: https://github.com/pytorch/pytorch/issues/110213 - causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) - - return causal_mask - - class MistralModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version("<", "4.48.0"): - # apply fix https://github.com/huggingface/transformers/commit/57d7594a79a9f5d835abf2d4d384db0e4818e548 - self._model.model._orig_update_causal_mask = self._model.model._update_causal_mask - self._model.model._update_causal_mask = types.MethodType(_mistral_update_causal_mask, self._model.model) if hasattr(self._model, "model") and hasattr(self._model.model, "layers"): for layer in self._model.model.layers: @@ -961,10 +799,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.48.0"): - self._model.model._update_causal_mask = self._model.model._orig_update_causal_mask - del self._model.model._orig_update_causal_mask - if hasattr(self._model.model, "model") and hasattr(self._model.model.model, "layers"): for layer in self._model.model.layers: if hasattr(layer.self_attn, "rotary_emb"): @@ -1385,17 +1219,9 @@ def _mpt_sdpa_attention_forward( if past_key_value is not None: # starting from v4.54 https://github.com/huggingface/transformers/blob/v4.54.0/src/transformers/models/mpt/modeling_mpt.py#L362 - if is_transformers_version(">=", "4.54"): - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - pkv_seq_length = past_key_value.get_seq_length() - - else: - if len(past_key_value) != 0: - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - past_key_value = (key_states, value_states) - pkv_seq_length = past_key_value[0].shape[2] + cache_kwargs = {"cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + pkv_seq_length = past_key_value.get_seq_length() key_length = key_states.shape[-2] query_length = seq_length if past_key_value is None else seq_length + pkv_seq_length @@ -1424,9 +1250,6 @@ def _mpt_sdpa_attention_forward( outputs = (attn_output, None) - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - return outputs @@ -1476,9 +1299,6 @@ def _mpt_block_forward( output = self.ffn(layernorm_output, residual) outputs = (output,) - if use_cache and is_transformers_version("<", "4.54"): - outputs += (attn_out[2],) - if output_attentions: outputs += (attn_out[1],) @@ -1622,133 +1442,6 @@ def __exit__(self, exc_type, exc_value, traceback): block.attention.forward = block.attention._orig_forward -def phi3_442_forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - **kwargs, -) -> Union[Tuple, BaseModelOutputWithPast]: - from transformers.cache_utils import Cache - from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask - - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - use_cache = use_cache if use_cache is not None else self.config.use_cache - - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - # retrieve input_ids and inputs_embeds - if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif input_ids is not None: - batch_size, seq_length = input_ids.shape[:2] - elif inputs_embeds is not None: - batch_size, seq_length = inputs_embeds.shape[:2] - else: - raise ValueError("You have to specify either input_ids or inputs_embeds") - - past_key_values_length = 0 - - if use_cache: - use_legacy_cache = not isinstance(past_key_values, Cache) - if use_legacy_cache: - if is_transformers_version("<", "5"): - past_key_values = DynamicCache.from_legacy_cache(past_key_values) - else: - past_key_values = DynamicCache(past_key_values) - - past_key_values_length = past_key_values.get_usable_length(seq_length) - - if position_ids is None: - device = input_ids.device if input_ids is not None else inputs_embeds.device - position_ids = torch.arange( - past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device - ) - position_ids = position_ids.unsqueeze(0).view(-1, seq_length) - else: - position_ids = position_ids.view(-1, seq_length).long() - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache: - is_padding_right = attention_mask[:, -1].sum().item() != batch_size - if is_padding_right: - raise ValueError( - "You are attempting to perform batched generation with padding_side='right'" - " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to " - " call `tokenizer.padding_side = 'left'` before tokenizing the input. " - ) - - if self._attn_implementation == "flash_attention_2": - # 2d mask is passed through the layers - attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None - else: - # 4d mask is passed through the layers - attention_mask = _prepare_4d_causal_attention_mask( - attention_mask, - (batch_size, seq_length), - inputs_embeds, - past_key_values_length, - sliding_window=self.config.sliding_window, - ) - - hidden_states = inputs_embeds - - # decoder layers - all_hidden_states = () if output_hidden_states else None - all_self_attns = () if output_attentions else None - next_decoder_cache = None - - for decoder_layer in self.layers: - if output_hidden_states: - all_hidden_states += (hidden_states,) - else: - layer_outputs = decoder_layer( - hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_value=past_key_values, - output_attentions=output_attentions, - use_cache=use_cache, - ) - - hidden_states = layer_outputs[0] - - if use_cache: - next_decoder_cache = layer_outputs[2 if output_attentions else 1] - - if output_attentions: - all_self_attns += (layer_outputs[1],) - - hidden_states = self.norm(hidden_states) - - # add hidden states from the last decoder layer - if output_hidden_states: - all_hidden_states += (hidden_states,) - - next_cache = None - if use_cache: - next_cache = postprocess_past_key_values(next_decoder_cache) if use_legacy_cache else next_decoder_cache - if not return_dict: - return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=next_cache, - hidden_states=all_hidden_states, - attentions=all_self_attns, - ) - - # Adapted from https://github.com/huggingface/transformers/blob/ccdabc5642bf84849af93f591e207dc625c8e1e1/src/transformers/models/phi3/modeling_phi3.py#L729 def _phi3_self_attn_sdpa_forward( self, @@ -1834,19 +1527,11 @@ def __enter__(self): ): self._model.config.max_position_embeddings = self._model.config.original_max_position_embeddings - if is_transformers_version("<", "4.48.0"): - self._model.model._orig_forward = self._model.model.forward - self._model.model.forward = types.MethodType(phi3_442_forward, self._model.model) - # https://github.com/huggingface/transformers/blob/30ee508c6c92a1c0aa0281d193c7c0fb815b8d2f/src/transformers/models/phi3/modeling_phi3.py#L113 # init inv_freq for torchscript tracing # 4.48 transformers version phi3 fixed, but issue still visible with trust_remote_true=True (trust_remote_code has _support_sdpa = False) for layer in self._model.model.layers: - if ( - is_torch_version(">=", "2.1.0") - and is_transformers_version("<", "4.48.0") - or not getattr(self._model, "_supports_sdpa", False) - ): + if not getattr(self._model, "_supports_sdpa", False): orig_self_attn_fwd = layer.self_attn.forward layer.self_attn.forward = types.MethodType(_phi3_self_attn_sdpa_forward, layer.self_attn) layer.self_attn._orig_forward = orig_self_attn_fwd @@ -2356,10 +2041,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask_original"): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - for layer in self._model.transformer.h: if hasattr(layer.attn, "_orig_attn"): layer.attn._attn = layer.attn._orig_attn @@ -2544,126 +2225,6 @@ def __exit__(self, exc_type, exc_value, traceback): block.norm_attn_norm.attn.rotary_emb.forward = block.norm_attn_norm.attn.rotary_emb._orig_forward -# Adapted from https://github.com/huggingface/transformers/blob/v4.41.0/src/transformers/models/persimmon/modeling_persimmon.py#L264 -def _persimmon_self_attn_sdpa_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional["Cache"] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - from transformers.models.persimmon.modeling_persimmon import apply_rotary_pos_emb - - if output_attentions: - return self._orig_forward( - hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache - ) - - bsz, q_len, _ = hidden_states.size() - - # [batch_size, seq_length, 3 x hidden_size] - fused_qkv = self.query_key_value(hidden_states) - - # 3 x [batch_size, seq_length, num_heads, head_dim] - (query_states, key_states, value_states) = self._split_heads(fused_qkv) - - if self.qk_layernorm: - query_states = self.q_layernorm(query_states) - key_states = self.k_layernorm(key_states) - - # [batch_size, num_heads, seq_length, head_dim] -> [batch_size, seq_length, num_heads, head_dim] - query_states = query_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - - if position_embeddings is None: - log.warning( - "The attention layers in this model are transitioning from computing the RoPE embeddings internally " - "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " - "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " - "removed and `position_embeddings` will be mandatory." - ) - cos, sin = self.rotary_emb(value_states, position_ids) - else: - cos, sin = position_embeddings - - rotary_ndims = self.rotary_ndims - # Partial rotary embedding - query_rot, query_pass = ( - query_states[..., :rotary_ndims], - query_states[..., rotary_ndims:], - ) - key_rot, key_pass = ( - key_states[..., :rotary_ndims], - key_states[..., rotary_ndims:], - ) - # [batch_size, seq_length, num_heads, head_dim // config.partial_rotary_factor] - query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) - - # [batch_size, seq_length, num_heads, head_dim] - query_states = torch.cat((query_rot, query_pass), dim=-1) - key_states = torch.cat((key_rot, key_pass), dim=-1) - - if past_key_value is not None: - # Specific to RoPE models with partial rotation - cache_kwargs = { - "sin": sin, - "cos": cos, - "partial_rotation_size": rotary_ndims, - "cache_position": cache_position, - } - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - causal_mask = attention_mask - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - - attn_output = F.scaled_dot_product_attention( - query_states, - key_states, - value_states, - causal_mask, - scale=1 / math.sqrt(self.head_dim), - dropout_p=self.attention_dropout.p, - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - attn_output = self.dense(attn_output) - - outputs = (attn_output, None) - - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - - return outputs - - -class PersimmonModelPatcher(OVDecoderModelPatcher): - def __enter__(self): - super().__enter__() - - if is_transformers_version("<", "4.56"): - for layer in self._model.model.layers: - if is_torch_version(">=", "2.1.0"): - orig_self_attn_fwd = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(_persimmon_self_attn_sdpa_forward, layer.self_attn) - layer.self_attn._orig_forward = orig_self_attn_fwd - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - - if is_transformers_version("<", "4.56"): - for layer in self._model.model.layers: - if hasattr(layer.self_attn, "_orig_forward"): - layer.self_attn.forward = layer.self_attn._orig_forward - - def _jais_attn_forward( self, hidden_states: Optional[Tuple[torch.FloatTensor]], @@ -2819,144 +2380,14 @@ def _falcon_prepare_4d_causal_attention_mask_with_cache_position( return causal_mask -def _falcon_update_causal_mask( - self, - attention_mask: torch.Tensor, - input_tensor: torch.Tensor, - cache_position: torch.Tensor, - past_key_values: "Cache", - output_attentions: bool, - head_mask: torch.Tensor, - alibi: torch.Tensor, -): - # copied from https://github.com/huggingface/transformers/blob/a30c865f991dfec9452cc64bd9a97bfbb96be036/src/transformers/models/falcon/modeling_falcon.py#L1130 - from transformers.cache_utils import StaticCache - from transformers.modeling_attn_mask_utils import AttentionMaskConverter - - # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static - # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. - # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using - # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 - - if self.config._attn_implementation == "flash_attention_2": - if attention_mask is not None and 0.0 in attention_mask: - return attention_mask - return None - - # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in - # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail - # to infer the attention mask. - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - using_static_cache = isinstance(past_key_values, StaticCache) - - # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward - if ( - self.config._attn_implementation == "sdpa" - and not using_static_cache - and not output_attentions - and head_mask is None - and alibi is None - ): - if AttentionMaskConverter._ignore_causal_mask_sdpa( - attention_mask, - inputs_embeds=input_tensor, - past_key_values_length=past_seen_tokens, - is_training=self.training, - ): - return None - - dtype, device = input_tensor.dtype, input_tensor.device - # difference from original, replace torch.finfo(dtype).min to float16 for prevent overflow for fp16/bf16 execution - min_dtype = torch.finfo(torch.float16).min - batch_size, sequence_length, _ = input_tensor.shape - if using_static_cache: - target_length = past_key_values.get_max_length() - else: - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else past_seen_tokens + sequence_length - ) - - # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). - causal_mask = _falcon_prepare_4d_causal_attention_mask_with_cache_position( - attention_mask, - sequence_length=sequence_length, - target_length=target_length, - dtype=dtype, - device=device, - min_dtype=min_dtype, - cache_position=cache_position, - batch_size=input_tensor.shape[0], - ) - - # We take care to integrate alibi bias in the causal_mask here - if head_mask is None and alibi is not None: - alibi = alibi.reshape(batch_size, -1, *alibi.shape[1:]) - causal_mask = torch.masked_fill( - alibi / math.sqrt(self.config.hidden_size // self.num_heads), - causal_mask < -1, - min_dtype, - ) - - if ( - self.config._attn_implementation == "sdpa" - and attention_mask is not None - and attention_mask.device.type == "cuda" - and not output_attentions - ): - # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when - # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. - # Details: https://github.com/pytorch/pytorch/issues/110213 - causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) - - return causal_mask - - class FalconModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() patch_cos_sin_cached_fp32(self._model.transformer) - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask"): - self._model.transformer._update_causal_mask_original = self._model.transformer._update_causal_mask - self._model.transformer._update_causal_mask = types.MethodType( - _falcon_update_causal_mask, self._model.transformer - ) - def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask_original"): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - - -class GptNeoxModelPatcher(OVDecoderModelPatcher): - def __enter__(self): - super().__enter__() - - if ( - is_transformers_version("<", "4.53") - and hasattr(self._model, "transformer") - and hasattr(self._model.transformer, "_update_causal_mask") - ): - self._model.transformer._update_causal_mask_original = self._model.transformer._update_causal_mask - self._model.transformer._update_causal_mask = types.MethodType( - _falcon_update_causal_mask, self._model.transformer - ) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - - if ( - is_transformers_version("<", "4.53") - and hasattr(self._model, "transformer") - and hasattr(self._model.transformer, "_update_causal_mask_original") - ): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - # Adopted from https://github.com/huggingface/optimum/blob/v1.24.0/optimum/bettertransformer/models/attention.py#L96 def _gptj_attn(self, query, key, value, attention_mask=None, head_mask=None): @@ -3506,22 +2937,6 @@ def __enter__(self): ): self._model.config._orig_attn_implementation = self._model.config._attn_implementation self._model.config._attn_implementation = "sdpa" - if self._model.config.model_type == "qwen2" and is_transformers_version("<", "4.48"): - from transformers.models.qwen2.modeling_qwen2 import QWEN2_ATTENTION_CLASSES - - sdpa_attn = QWEN2_ATTENTION_CLASSES["sdpa"] - - for layer in self._model.model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) - - if self._model.config.model_type == "llama" and is_transformers_version("<", "4.47"): - from transformers.models.llama.modeling_llama import LLAMA_ATTENTION_CLASSES - - sdpa_attn = LLAMA_ATTENTION_CLASSES["sdpa"] - for layer in self._model.model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) if self._internal_patcher is not None: return self._internal_patcher.__enter__() @@ -4472,115 +3887,69 @@ def __exit__(self, exc_type, exc_value, traceback): def patch_qwen2vl_vision_blocks(model, force_new_behaviour=False): - if not force_new_behaviour and is_transformers_version("<=", "4.48.99"): - # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L390 - # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) - def sdpa_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - rotary_pos_emb: torch.Tensor = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> torch.Tensor: - from transformers.models.qwen2_vl.modeling_qwen2_vl import apply_rotary_pos_emb_vision - - seq_length = hidden_states.shape[0] - q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - - if is_transformers_version(">=", "4.49"): - if position_embeddings is None: - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - cos = emb.cos().float() - sin = emb.sin().float() - else: - cos, sin = position_embeddings - q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) - else: - q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) - k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) - - q = q.transpose(0, 1) - k = k.transpose(0, 1) - v = v.transpose(0, 1) - attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) - attn_output = attn_output.transpose(0, 1) - attn_output = attn_output.reshape(seq_length, -1) - attn_output = self.proj(attn_output) - return attn_output - - # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L430 - # added attention_mask input propagation to self.attn - def block_forward(self, hidden_states, attention_mask, rotary_pos_emb) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), attention_mask=attention_mask, rotary_pos_emb=rotary_pos_emb - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - else: - # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L391 - # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) - def sdpa_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - rotary_pos_emb: torch.Tensor = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ): - def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - seq_length = hidden_states.shape[0] - q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - if position_embeddings is None: - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - cos = emb.cos().float() - sin = emb.sin().float() - else: - cos, sin = position_embeddings - q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) - q = q.transpose(0, 1) - k = k.transpose(0, 1) - v = v.transpose(0, 1) - attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) - attn_output = attn_output.transpose(0, 1) - attn_output = attn_output.reshape(seq_length, -1) - attn_output = self.proj(attn_output) - return attn_output - - # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L446 - # added attention_mask input propagation to self.attn - def block_forward( - self, - hidden_states, - attention_mask, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - attention_mask=attention_mask, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states + # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L391 + # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) + def sdpa_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + rotary_pos_emb: torch.Tensor = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ): + def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + seq_length = hidden_states.shape[0] + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + if position_embeddings is None: + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) + cos = emb.cos().float() + sin = emb.sin().float() + else: + cos, sin = position_embeddings + q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) + attn_output = attn_output.transpose(0, 1) + attn_output = attn_output.reshape(seq_length, -1) + attn_output = self.proj(attn_output) + return attn_output + + # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L446 + # added attention_mask input propagation to self.attn + def block_forward( + self, + hidden_states, + attention_mask, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + attention_mask=attention_mask, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states for block in model.blocks: block._orig_forward = block.forward @@ -4869,58 +4238,41 @@ def patched_forward(*args, **kwargs): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.53.0"): - # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min - # to avoid overflow issues on some hardware (e.g. Intel NPU) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) + # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min + # to avoid overflow issues on some hardware (e.g. Intel NPU) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) - # for decoder models, we use eager mask without vmap for sdpa as well - # to avoid a nan output issue in OpenVINO that only happens in case of: - # non-stateful models on cpu and stateful models on npu - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # for decoder models, we use eager mask without vmap for sdpa as well + # to avoid a nan output issue in OpenVINO that only happens in case of: + # non-stateful models on cpu and stateful models on npu + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) class SanaTextEncoderModelPatcher(ModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version("<", "4.47.0"): - from transformers.models.gemma2.modeling_gemma2 import GEMMA2_ATTENTION_CLASSES - - sdpa_attn = GEMMA2_ATTENTION_CLASSES["sdpa"] - for layer in self._model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) - else: - self._model.config._orig_attn_implementation = self._model.config._attn_implementation - self._model.config._attn_implementation = "sdpa" + self._model.config._orig_attn_implementation = self._model.config._attn_implementation + self._model.config._attn_implementation = "sdpa" - if is_transformers_version(">=", "4.53"): - # starting from 4.53, we get unmatching outputs if we use the boolean mask - # TODO: This is an openvino issue (inconsistency between boolean and float masks) - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # starting from 4.53, we get unmatching outputs if we use the boolean mask + # TODO: This is an openvino issue (inconsistency between boolean and float masks) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.47.0"): - for layer in self._model.layers: - layer.self_attn.forward = layer.self_attn._orig_forward - del layer.self_attn._orig_forward - else: - self._model.config._attn_implementation = self._model.config._orig_attn_implementation - del self._model.config._orig_attn_implementation + self._model.config._attn_implementation = self._model.config._orig_attn_implementation + del self._model.config._orig_attn_implementation - if is_transformers_version(">=", "4.53"): - # remove the eager_mask_without_vmap from the ALL_MASK_ATTENTION_FUNCTIONS - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + # remove the eager_mask_without_vmap from the ALL_MASK_ATTENTION_FUNCTIONS + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) class MiniCPMModelPatcher(OVDecoderModelPatcher): @@ -4972,56 +4324,6 @@ def patched_forward(*args, **kwargs): self.patched_forward = patched_forward -# Adopted from https://github.com/huggingface/transformers/blob/v4.49.0-Gemma-3/src/transformers/models/gemma3/modeling_gemma3.py#L1147 -def _gemma3_mm_update_causal_mask( - self, attention_mask, token_type_ids, past_key_values, cache_position, input_tensor, is_training: bool = False -): - if attention_mask is not None and attention_mask.dim() == 4: - # In this case we assume that the mask comes already in inverted - # form and requires no inversion or slicing. - return attention_mask - - min_dtype = torch.finfo(torch.float16).min - inputs_lead_dim, sequence_length = input_tensor.shape[:2] - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else cache_position[0] + sequence_length + 1 - ) - - causal_mask = torch.full( - (sequence_length, target_length), fill_value=min_dtype, dtype=self.dtype, device=cache_position.device - ) - - # Causal diagonal mask only if training, otherwise attend to the whole prefix. Training-specific attn for prefix is handled below - if sequence_length != 1: - causal_mask = torch.triu(causal_mask, diagonal=1) - - causal_mask *= torch.arange(target_length, device=cache_position.device) > cache_position.reshape(-1, 1) - causal_mask = causal_mask[None, None, :, :].expand(inputs_lead_dim, 1, -1, -1) - - # Apply bidirectional mask on images if token type ids are provided - if token_type_ids is not None and sequence_length != 1: - token_type_mask = token_type_ids.unsqueeze(1) == token_type_ids.unsqueeze(2) - token_type_mask[token_type_ids == 0] = False # if text token do not change anything - token_type_mask = token_type_mask.unsqueeze(1).to(causal_mask.device, dtype=torch.bool) - causal_mask = causal_mask.clone() - causal_mask[:, :, :, :sequence_length] = causal_mask[:, :, :, :sequence_length].masked_fill( - token_type_mask, 0.0 - ) - - if attention_mask is not None: - causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit - mask_length = attention_mask.shape[-1] - - # Then apply padding mask (will mask pad tokens) - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(causal_mask.device) - padding_mask = padding_mask == 0 - causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(padding_mask, min_dtype) - - return causal_mask - - class Gemma3LMModelPatcher(OVDecoderModelPatcher): def __init__( self, @@ -5047,12 +4349,7 @@ def forward( ) forward_kwargs = {} - if is_transformers_version("<", "4.52"): - attention_mask = self._update_causal_mask_mm( - attention_mask, token_type_ids, past_key_values, cache_position, inputs_embeds - ) - else: - forward_kwargs["token_type_ids"] = token_type_ids + forward_kwargs["token_type_ids"] = token_type_ids result = self.__orig_forward( input_ids=None, @@ -5068,41 +4365,14 @@ def forward( result["past_key_values"] = postprocess_past_key_values(upd_pkv) return result - if is_transformers_version("<", "4.53.0"): - model.__orig_forward = model.forward - model.forward = types.MethodType(forward, model) - super().__init__(config, model, model_kwargs) def __enter__(self): super().__enter__() - if is_transformers_version("<", "4.52.0"): - self._model._update_causal_mask_mm = types.MethodType(_gemma3_mm_update_causal_mask, self._model) - elif ( - is_transformers_version("<", "4.53.0") - and hasattr(self._model, "model") - and hasattr(self._model.model, "_update_causal_mask") - ): - self._model.model._orig_update_causual_mask = self._model.model._update_causal_mask - self._model.model._update_causal_mask = types.MethodType(_gemma3_mm_update_causal_mask, self._model.model) - def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53.0"): - self._model.forward = self._model.__orig_forward - - if is_transformers_version("<", "4.52"): - del self._update_causal_mask_mm - elif ( - is_transformers_version("<", "4.53.0") - and hasattr(self._model, "model") - and hasattr(self._model.model, "_orig_update_causual_mask") - ): - self._model.model._update_causal_mask = self._model.model._orig_update_causual_mask - del self._model.model._orig_update_causual_mask - # Creates a dict of causal masks with bidirectional attention for vision tokens # on sliding_attention layers, matching the behavior of transformers @@ -5565,14 +4835,10 @@ def embeddings_forward( nb_patches_h = p_attn_mask[:, 0].sum() nb_patches_w = p_attn_mask[0].sum() - if is_transformers_version("<", "4.55"): - fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h) - fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w) - else: - h_indices = torch.arange(nb_patches_h, device=pixel_values.device, dtype=pixel_values.dtype) - w_indices = torch.arange(nb_patches_w, device=pixel_values.device, dtype=pixel_values.dtype) - fractional_coords_h = h_indices / nb_patches_h * (1 - 1e-6) - fractional_coords_w = w_indices / nb_patches_w * (1 - 1e-6) + h_indices = torch.arange(nb_patches_h, device=pixel_values.device, dtype=pixel_values.dtype) + w_indices = torch.arange(nb_patches_w, device=pixel_values.device, dtype=pixel_values.dtype) + fractional_coords_h = h_indices / nb_patches_h * (1 - 1e-6) + fractional_coords_w = w_indices / nb_patches_w * (1 - 1e-6) bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) @@ -5658,202 +4924,6 @@ def __exit__(self, exc_type, exc_value, traceback): layer.self_attn.forward = layer.self_attn._orig_forward -# Adopted from https://github.com/huggingface/optimum/blob/main/optimum/bettertransformer/models/decoder_models.py#L367 -def _blenderbot_attn_forward_legacy( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if output_attentions or layer_head_mask is not None: - return self._orig_forward( - hidden_states, key_value_states, past_key_value, attention_mask, layer_head_mask, output_attentions - ) - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) - # get key, value proj - # `past_key_value[0].shape[2] == key_value_states.shape[1]` - # is checking that the `sequence_length` of the `past_key_value` is the same as - # the provided `key_value_states` to support prefix tuning - if is_cross_attention and past_key_value is not None and past_key_value[0].shape[2] == key_value_states.shape[1]: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - query_states = self._shape(query_states, tgt_len, bsz) - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.dropout if self.training else 0.0, - is_causal=False, - ) - - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, None, past_key_value - - -# Adopted from https://github.com/huggingface/transformers/blob/v4.52.3/src/transformers/models/blenderbot/modeling_blenderbot.py#L156 -def _blenderbot_attn_forward_new( - self, - hidden_states: torch.Tensor, - key_value_states=None, - past_key_value=None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, - cache_position: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - if output_attentions or layer_head_mask is not None: - return self._orig_forward( - hidden_states, - key_value_states, - past_key_value, - attention_mask, - layer_head_mask, - output_attentions, - cache_position, - ) - is_cross_attention = key_value_states is not None - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states).view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - query_states = query_states - - if past_key_value is not None: - if isinstance(past_key_value, EncoderDecoderCache): - is_updated = past_key_value.is_updated.get(self.layer_idx) - if is_cross_attention: - # after the first generated id, we can subsequently re-use all key/value_states from cache - curr_past_key_value = past_key_value.cross_attention_cache - else: - curr_past_key_value = past_key_value.self_attention_cache - else: - curr_past_key_value = past_key_value - - current_states = key_value_states if is_cross_attention else hidden_states - if is_cross_attention and past_key_value is not None and is_updated: - # reuse k,v, cross_attentions - key_states = curr_past_key_value.key_cache[self.layer_idx] - value_states = curr_past_key_value.value_cache[self.layer_idx] - else: - key_states = self.k_proj(current_states) - value_states = self.v_proj(current_states) - key_states = key_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - - if past_key_value is not None: - # save all key/value_states to cache to be re-used for fast auto-regressive generation - cache_position = cache_position if not is_cross_attention else None - key_states, value_states = curr_past_key_value.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) - # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls - if is_cross_attention: - past_key_value.is_updated[self.layer_idx] = True - - proj_shape = (bsz, self.num_heads, -1, self.head_dim) - # difference with original, removed query_states = query_states.reshape(*proj_shape) * self.scale as scale is part of SDPA - query_states = query_states.reshape(*proj_shape) - key_states = key_states.reshape(*proj_shape) - value_states = value_states.reshape(*proj_shape) - - # Difference with original, use SDPA instead of eager attention - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.dropout if self.training else 0.0, - is_causal=False, - ) - - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - outputs = (attn_output, None) - - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - - return outputs - - -if is_transformers_version(">=", "4.52"): - _blenderbot_attn_forward = _blenderbot_attn_forward_new -else: - _blenderbot_attn_forward = _blenderbot_attn_forward_legacy - - def modulewise_patch(model, module_cls, patch_forward): for _, module in model.named_children(): if isinstance(module, module_cls): @@ -5875,54 +4945,6 @@ def modulewise_unpatch(model, module_cls): modulewise_unpatch(module, module_cls) -class BlenderbotModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot.modeling_blenderbot import BlenderbotAttention - - modulewise_patch(self._model, BlenderbotAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot.modeling_blenderbot import BlenderbotAttention - - modulewise_unpatch(self._model, BlenderbotAttention) - - -class BlenderbotSmallModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot_small.modeling_blenderbot_small import BlenderbotSmallAttention - - modulewise_patch(self._model, BlenderbotSmallAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot_small.modeling_blenderbot_small import BlenderbotSmallAttention - - modulewise_unpatch(self._model, BlenderbotSmallAttention) - - -class PegasusModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.pegasus.modeling_pegasus import PegasusAttention - - modulewise_patch(self._model, PegasusAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.pegasus.modeling_pegasus import PegasusAttention - - modulewise_unpatch(self._model, PegasusAttention) - - # Copied from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py#L596 # In 4.52.0, the loop is only over hitted experts, but we need to loop over all experts for tracing def _qwen2moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -5974,44 +4996,26 @@ class Qwen2MoEPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.52.0"): - if is_transformers_version("<", "5"): - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock + if is_transformers_version("<", "5"): + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - modulewise_patch(self._model, Qwen2MoeSparseMoeBlock, _qwen2moe_sparse_block_forward) - else: - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts + modulewise_patch(self._model, Qwen2MoeSparseMoeBlock, _qwen2moe_sparse_block_forward) + else: + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - self.original_moe_forward = Qwen2MoeExperts.forward - Qwen2MoeExperts.forward = lfm2_moe_experts_forward + self.original_moe_forward = Qwen2MoeExperts.forward + Qwen2MoeExperts.forward = lfm2_moe_experts_forward def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.52.0"): - if is_transformers_version("<", "5"): - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - - modulewise_unpatch(self._model, Qwen2MoeSparseMoeBlock) - else: - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - - Qwen2MoeExperts.forward = self.original_moe_forward - - -class MarianModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version(">=", "4.49.0") and is_transformers_version("<", "4.56"): - from transformers.models.marian.modeling_marian import MarianAttention - - modulewise_patch(self._model, MarianAttention, _blenderbot_attn_forward) + if is_transformers_version("<", "5"): + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.49.0") and is_transformers_version("<", "4.56"): - from transformers.models.marian.modeling_marian import MarianAttention + modulewise_unpatch(self._model, Qwen2MoeSparseMoeBlock) + else: + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - modulewise_unpatch(self._model, MarianAttention) + Qwen2MoeExperts.forward = self.original_moe_forward # Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L698 @@ -6041,210 +5045,6 @@ def speecht5_decoder_prenet_forward( return inputs_embeds -# Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L889 -# this is a patch to avoid CPU plugin issue that is happened on 16-th iteration of token generation -# values computed by self-attention attn_output = torch.bmm(attn_probs, value_states) in a decoder gets incorrect -def speecht5_attention_forward( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - position_bias: Optional[torch.Tensor] = None, - output_attentions: bool = False, - serialize: bool = False, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - is_cross_attention = key_value_states is not None - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) * self.scaling - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_states = value_states.view(*proj_shape) - - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" - f" {attn_weights.size()}" - ) - - # relative attention bias - if position_bias is not None: - reshape_q = query_states.contiguous().view(bsz * self.num_heads, -1, self.head_dim).transpose(0, 1) - rel_pos_bias = torch.matmul(reshape_q, position_bias.transpose(-2, -1)) - rel_pos_bias = rel_pos_bias.transpose(0, 1).view( - bsz * self.num_heads, position_bias.size(0), position_bias.size(1) - ) - attn_weights += rel_pos_bias - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1) - - if layer_head_mask is not None: - if layer_head_mask.size() != (self.num_heads,): - raise ValueError( - f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" - ) - attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if output_attentions: - # this operation is a bit awkward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to be reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None - - attn_probs = torch.nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) - - # this is a patch to avoid CPU plugin issue!!! - # issue is happened on 16-th iteration of token generation - # since 16-th iteration of token generation, values computed by self-attention in a decoder gets incorrect - eps = 1e-30 - attn_output = torch.bmm(attn_probs + eps, value_states) - - if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned across GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights_reshaped, past_key_value - - -# Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L1121 -# this is a patch for a model to avoid incorrect tracing -# cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple are computed using encoder_hidden_states -def speecht5_decoder_layer_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - encoder_hidden_states: Optional[torch.Tensor] = None, - encoder_attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - cross_attn_layer_head_mask: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - output_attentions: Optional[bool] = False, - use_cache: Optional[bool] = True, - serialize: bool = False, -): - residual = hidden_states - - # Self Attention - # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 - self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None - # add present self-attn cache to positions 1,2 of present_key_value tuple - hidden_states, self_attn_weights, present_key_value = self.self_attn( - hidden_states=hidden_states, - past_key_value=self_attn_past_key_value, - attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - output_attentions=output_attentions, - serialize=serialize, - ) - - hidden_states = self.dropout(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - # Cross-Attention Block - cross_attn_present_key_value = None - cross_attn_weights = None - if encoder_hidden_states is not None: - residual = hidden_states - - # this is a patch for a model to avoid incorrect tracing!!! - # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple - # are computed using encoder_hidden_states - if past_key_value is not None and len(past_key_value) > 3: - cross_attn_past_key_value = past_key_value[-2:] - else: - cross_attn_past_key_value = None - hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn( - hidden_states=hidden_states, - key_value_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - layer_head_mask=cross_attn_layer_head_mask, - past_key_value=cross_attn_past_key_value, - output_attentions=output_attentions, - ) - hidden_states = self.dropout(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.encoder_attn_layer_norm(hidden_states) - - # add cross-attn to positions 3,4 of present_key_value tuple - present_key_value = present_key_value + cross_attn_present_key_value - - # Fully Connected - hidden_states = hidden_states + self.feed_forward(hidden_states) - hidden_states = self.final_layer_norm(hidden_states) - - outputs = (hidden_states,) - - if output_attentions: - outputs += (self_attn_weights, cross_attn_weights) - - if use_cache: - outputs += (present_key_value,) - - return outputs - - class SpeechT5ModelPatcher(ModelPatcher): def __enter__(self): if self.real_config._behavior != "vocoder": @@ -6254,12 +5054,6 @@ def __enter__(self): self._model.speecht5.decoder.prenet.forward = types.MethodType( speecht5_decoder_prenet_forward, self._model.speecht5.decoder.prenet ) - if is_transformers_version("<", "4.54"): - for layer in self._model.speecht5.decoder.wrapped_decoder.layers: - layer.__orig_forward = layer.forward - layer.forward = types.MethodType(speecht5_decoder_layer_forward, layer) - layer.self_attn.__orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(speecht5_attention_forward, layer.self_attn) def __exit__(self, exc_type, exc_value, traceback): if self.real_config._behavior != "vocoder": @@ -6268,10 +5062,6 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.speecht5.decoder.prenet.forward = types.MethodType( self._model.speecht5.decoder.prenet.__orig_forward, self._model.speecht5.decoder.prenet ) - if is_transformers_version("<", "4.54"): - for layer in self._model.speecht5.decoder.wrapped_decoder.layers: - layer.forward = types.MethodType(layer.__orig_forward, layer) - layer.self_attn.forward = types.MethodType(layer.self_attn.__orig_forward, layer.self_attn) def __init__( self, @@ -6950,44 +5740,6 @@ def llama4_attn_forward( return attn_output, attn_weights -# modified from https://github.com/huggingface/transformers/blob/v4.51.0/src/transformers/models/llama4/modeling_llama4.py#L157 -# due to openvino transformations issue removed routed_out.view(-1, hidden_dim) in scatter_add_ -def llama4_moe_forward(self, hidden_states): - batch, seq_len, hidden_dim = hidden_states.shape - hidden_states = hidden_states.view(-1, self.hidden_dim) - router_logits = self.router(hidden_states).transpose(0, 1) - tokens_per_expert = batch * seq_len - - router_top_value, router_indices = torch.topk(router_logits.transpose(0, 1), self.top_k, dim=1) - router_scores = ( - torch.full_like(router_logits.transpose(0, 1), float("-inf")) - .scatter_(1, router_indices, router_top_value) - .transpose(0, 1) - ) - # We do this to make sure we have -inf for non topK tokens before going through the ! - # Here we are just creating a tensor to index each and every single one of the hidden states. Let s maybe register a buffer for this! - router_indices = ( - torch.arange(tokens_per_expert, device=hidden_states.device).view(1, -1).expand(router_scores.size(0), -1) - ) - router_scores = torch.sigmoid(router_scores.float()).to(hidden_states.dtype) - - router_indices = router_indices.reshape(-1, 1).expand(-1, hidden_dim) - routed_in = torch.gather( - input=hidden_states, - dim=0, - index=router_indices, - ).to(hidden_states.device) - # we gather inputs corresponding to each expert based on the router indices - routed_in = routed_in * router_scores.transpose(0, 1).reshape(-1, 1) - routed_out = self.experts(routed_in) - out = self.shared_expert(hidden_states) - # now that we finished expert computation -> we scatter add because we gathered previously - # we have to do this because we used all experts on all tokens. This is faster than the for loop, tho you are compute bound - # this scales a lot better if you do EP! - out.scatter_add_(dim=0, index=router_indices, src=routed_out) - return out, router_scores - - # Copied from https://github.com/huggingface/transformers/blob/v4.56.0/src/transformers/masking_utils.py#L105 # transformers.masking_utils._legacy_chunked_overlay deprecated since transformers v5 def _legacy_chunked_overlay(chunk_size: int) -> Callable: @@ -7004,9 +5756,6 @@ def __enter__(self): self._model.model.rotary_emb._orig_forward = self._model.model.rotary_emb.forward self._model.model.rotary_emb.forward = types.MethodType(llama4_rope_forward, self._model.model.rotary_emb) for layer in self._model.model.layers[: self._model.model.config.num_hidden_layers]: - if layer.is_moe_layer and is_transformers_version("<", "4.54"): - layer.feed_forward._orig_forward = layer.feed_forward.forward - layer.feed_forward.forward = types.MethodType(llama4_moe_forward, layer.feed_forward) layer.self_attn._orig_forward = layer.self_attn.forward layer.self_attn.forward = types.MethodType(llama4_attn_forward, layer.self_attn) @@ -7022,8 +5771,6 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.model.rotary_emb.forward = self._model.model.rotary_emb._orig_forward for layer in self._model.model.layers[: self._model.model.config.num_hidden_layers]: - if layer.is_moe_layer and is_transformers_version("<", "4.54"): - layer.feed_forward.forward = layer.feed_forward._orig_forward layer.self_attn.forward = layer.self_attn._orig_forward if is_transformers_version(">=", "4.56"): @@ -7359,26 +6106,24 @@ class Qwen3MoeModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.53"): - if is_transformers_version("<", "5"): - self.original_moe_forward = Qwen3MoeSparseMoeBlock.forward - Qwen3MoeSparseMoeBlock.forward = qwen3_moe_forward_patched - else: - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + if is_transformers_version("<", "5"): + self.original_moe_forward = Qwen3MoeSparseMoeBlock.forward + Qwen3MoeSparseMoeBlock.forward = qwen3_moe_forward_patched + else: + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts - self.original_moe_forward = Qwen3MoeExperts.forward - Qwen3MoeExperts.forward = lfm2_moe_experts_forward + self.original_moe_forward = Qwen3MoeExperts.forward + Qwen3MoeExperts.forward = lfm2_moe_experts_forward def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.53"): - if is_transformers_version("<", "5"): - Qwen3MoeSparseMoeBlock.forward = self.original_moe_forward - else: - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + if is_transformers_version("<", "5"): + Qwen3MoeSparseMoeBlock.forward = self.original_moe_forward + else: + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts - Qwen3MoeExperts.forward = self.original_moe_forward + Qwen3MoeExperts.forward = self.original_moe_forward # The original implementation of this forward method can be found at: @@ -8076,23 +6821,21 @@ class GptOssModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.55.0"): - if is_transformers_version("<", "5"): - from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts + if is_transformers_version("<", "5"): + from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts - self.original_gpt_oss_forward = GptOssExperts.forward - GptOssExperts.forward = gpt_oss_forward - else: - register_ov_batched_mm(self) + self.original_gpt_oss_forward = GptOssExperts.forward + GptOssExperts.forward = gpt_oss_forward + else: + register_ov_batched_mm(self) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.55.0"): - if is_transformers_version("<", "5"): - from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts + if is_transformers_version("<", "5"): + from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts - GptOssExperts.forward = self.original_gpt_oss_forward + GptOssExperts.forward = self.original_gpt_oss_forward # This patch overrides the following line in Transformers: diff --git a/optimum/exporters/openvino/patching_utils.py b/optimum/exporters/openvino/patching_utils.py index 2c3128b6bc..a50047c974 100644 --- a/optimum/exporters/openvino/patching_utils.py +++ b/optimum/exporters/openvino/patching_utils.py @@ -23,43 +23,24 @@ import transformers from transformers import PreTrainedModel from transformers.cache_utils import DynamicCache, EncoderDecoderCache +from transformers.masking_utils import ( + ALL_MASK_ATTENTION_FUNCTIONS, + _ignore_causal_mask_sdpa, + and_masks, + causal_mask_function, + eager_mask, + find_packed_sequence_indices, + padding_mask_function, + prepare_padding_mask, + sdpa_mask, +) from transformers.modeling_outputs import BaseModelOutput from optimum.exporters.base import ExporterConfig +from optimum.exporters.openvino._traceable_decorator import traceable_check_model_inputs from optimum.intel.utils.import_utils import is_transformers_version -if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - from optimum.exporters.openvino._traceable_cache import TraceableCache - - -if is_transformers_version(">=", "4.48"): - from transformers.cache_utils import DynamicCache, EncoderDecoderCache -if is_transformers_version(">=", "4.53"): - from transformers.masking_utils import ( - ALL_MASK_ATTENTION_FUNCTIONS, - _ignore_causal_mask_sdpa, - and_masks, - causal_mask_function, - eager_mask, - padding_mask_function, - prepare_padding_mask, - sdpa_mask, - ) - - -if is_transformers_version(">=", "4.53.1"): - from transformers.masking_utils import find_packed_sequence_indices - - -if is_transformers_version(">=", "4.54"): - from transformers.utils import TransformersKwargs - - from optimum.exporters.openvino._traceable_decorator import traceable_check_model_inputs -else: - TransformersKwargs = object - - if is_transformers_version(">=", "4.56"): import transformers.masking_utils from transformers.cache_utils import DynamicLayer @@ -86,18 +67,14 @@ def override_arguments(args, kwargs, forward_signature, model_kwargs: dict[str, def preprocess_encoder_outputs(encoder_outputs): - if is_transformers_version(">=", "4.54") and isinstance(encoder_outputs, (list, tuple)): + if isinstance(encoder_outputs, (list, tuple)): encoder_outputs = BaseModelOutput(*encoder_outputs) return encoder_outputs def preprocess_past_key_values(past_key_values): - if ( - is_transformers_version(">=", "4.48") - and isinstance(past_key_values, (list, tuple)) - and isinstance(past_key_values[0], (list, tuple)) - ): + if isinstance(past_key_values, (list, tuple)) and isinstance(past_key_values[0], (list, tuple)): if len(past_key_values[0]) == 2: if hasattr(DynamicCache, "from_legacy_cache"): past_key_values = DynamicCache.from_legacy_cache(past_key_values) @@ -162,12 +139,6 @@ def find_packed_sequence_indices_patched(position_ids: torch.Tensor) -> torch.Te return torch.zeros_like(position_ids) -if is_transformers_version(">=", "4.53"): - _prepare_padding_mask_slice = "_slice" in inspect.signature(prepare_padding_mask).parameters -else: - _prepare_padding_mask_slice = False - - # Custom vectorized implementation of sdpa_mask without using vmap def _orig_sdpa_mask_without_vmap( batch_size: int, @@ -185,7 +156,7 @@ def _orig_sdpa_mask_without_vmap( q_length = cache_position.shape[0] # Potentially pad the 2D mask, and slice it correctly - if _prepare_padding_mask_slice: + if "_slice" in inspect.signature(prepare_padding_mask).parameters: padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset, _slice=False) else: padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset) @@ -293,7 +264,7 @@ def __init__( self.orig_forward_name = "forward" if hasattr(self._model, "forward") else "call" self.orig_forward = getattr(self._model, self.orig_forward_name) - if is_transformers_version(">=", "4.54") and hasattr(self.orig_forward, "__wrapped__"): + if hasattr(self.orig_forward, "__wrapped__"): # the original check_model_inputs has some failing cases that we fix in traceable_check_model_inputs # we fix those issues in a PR in transformers https://github.com/huggingface/transformers/pull/40811 # issues are: support for positional args (use_cache for instance) and fix for _CAN_RECORD_REGISTRY @@ -396,25 +367,16 @@ def __enter__(self): self.patch_ops() setattr(self._model, self.orig_forward_name, self.patched_forward) - # This is a workaround for the Cache class in transformers, we replace it - # with traceable cache is because the original one used in transformers - # inherited from nn.Module (for a couple versions), which can't be traced as input. - if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - self.original_cache_class = transformers.cache_utils.Cache - transformers.cache_utils.Cache = TraceableCache - # This is a workaround for mask generation in transformers >= 4.53. # The masking process uses vmap which is not traceable by TorchScript. - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask_without_vmap) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask_without_vmap) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) # This is a workaround for the find_packed_sequence_indices function in transformers which # should only return a tensor of zeros with the same shape as position_ids indicating no packed sequence indices. # The function uses torch.diff which is not traceable by TorchScript. - if is_transformers_version(">=", "4.53.1"): - self.original_find_packed_sequence_indices = find_packed_sequence_indices - transformers.masking_utils.find_packed_sequence_indices = find_packed_sequence_indices_patched + self.original_find_packed_sequence_indices = find_packed_sequence_indices + transformers.masking_utils.find_packed_sequence_indices = find_packed_sequence_indices_patched # Starting from transformers 4.56.0, DynamicCache uses DynamicLayer which has an update method # that uses torch.cat to concatenate an empty tensor with the key/value states during the first call. @@ -427,15 +389,10 @@ def __exit__(self, exc_type, exc_value, traceback): self.restore_ops() setattr(self._model, self.orig_forward_name, self.orig_forward) - if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - transformers.cache_utils.Cache = self.original_cache_class - - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) - if is_transformers_version(">=", "4.53.1"): - transformers.masking_utils.find_packed_sequence_indices = self.original_find_packed_sequence_indices + transformers.masking_utils.find_packed_sequence_indices = self.original_find_packed_sequence_indices if is_transformers_version(">=", "4.56"): DynamicLayer.update = self.original_dynamic_layer_update diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 15ed8c94cc..afa1142c18 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -1465,7 +1465,9 @@ def prepare_inputs_for_generation( "you are calling `prepare_inputs_for_generation` directly with `use_cache=True`" ) # infer from cache_params: None means prefill (0), otherwise means decoding stage - cache_position = torch.tensor([0 if cache_params is None else 1], device=input_ids.device, dtype=torch.long) + cache_position = torch.tensor( + [0 if cache_params is None else 1], device=input_ids.device, dtype=torch.long + ) if cache_position[0] > 0: # decoding stage so it takes the last token input_ids = input_ids[:, -1].unsqueeze(-1) diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index ba25c0e413..3e3edf2129 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -25,6 +25,7 @@ from openvino._offline_transformations import apply_moc_transformations, compress_model_transformation from transformers import ( AutoConfig, + AutoModelForImageTextToText, AutoModelForSeq2SeqLM, AutoModelForSpeechSeq2Seq, GenerationConfig, @@ -55,18 +56,6 @@ ) -# AutoModelForVision2Seq is deprecated since v4.54 -# https://github.com/huggingface/transformers/blob/v4.54.0/src/transformers/models/auto/modeling_auto.py#L2151 -if is_transformers_version(">=", "4.54.0"): - from transformers import AutoModelForImageTextToText - - transformers_auto_class = AutoModelForImageTextToText -else: - from transformers import AutoModelForVision2Seq - - transformers_auto_class = AutoModelForVision2Seq - - core = Core() logger = logging.getLogger(__name__) @@ -1072,7 +1061,7 @@ def _reorder_cache( INPUTS_DOCSTRING, ) class OVModelForVision2Seq(OVModelForSeq2SeqLM): - auto_model_class = transformers_auto_class + auto_model_class = AutoModelForImageTextToText main_input_name = "pixel_values" export_feature = "image-to-text" @@ -1538,18 +1527,3 @@ def prepare_inputs_for_generation( "decoder_position_ids": decoder_position_ids, "cache_position": cache_position, } - - def _get_logits_processor(self, generation_config: GenerationConfig, *args, **kwargs): - # Whisper uses forced_decoder_ids for default task and language specification, while original _get_logits_processor does not allow it - # see for details https://github.com/huggingface/transformers/issues/37172 - if not hasattr(generation_config, "forced_decoder_ids") or is_transformers_version(">=", "4.53.0"): - # since transformers 4.53.0, forced_decoder_ids is deprecated: https://github.com/huggingface/transformers/pull/38232 - logits_processor = super()._get_logits_processor(generation_config, *args, **kwargs) - else: - forced_decoder_ids = generation_config.forced_decoder_ids - - if is_transformers_version(">=", "4.50.0"): - generation_config.forced_decoder_ids = None - logits_processor = super()._get_logits_processor(generation_config, *args, **kwargs) - generation_config.forced_decoder_ids = forced_decoder_ids - return logits_processor diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index d58fdb985e..59513e29fb 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -63,14 +63,7 @@ ) -if is_transformers_version(">=", "4.46.0"): - from transformers import AutoModelForImageTextToText - - transformers_auto_class = AutoModelForImageTextToText -else: - from transformers import AutoModelForVision2Seq - - transformers_auto_class = AutoModelForVision2Seq +from transformers import AutoModelForImageTextToText if TYPE_CHECKING: @@ -392,7 +385,7 @@ def forward(self, audio_feature, audio_mask): class OVModelForVisualCausalLM(OVBaseModel, GenerationMixin): export_feature = "image-text-to-text" additional_parts = [] - auto_model_class = transformers_auto_class + auto_model_class = AutoModelForImageTextToText @classproperty def _all_ov_model_paths(cls) -> Dict[str, str]: @@ -1167,7 +1160,7 @@ def preprocess_inputs( else: prompt = text - if is_transformers_version(">", "4.47.99") and getattr(processor, "patch_size", None) is None: + if getattr(processor, "patch_size", None) is None: if ( getattr(config, "vision_config", None) is not None and getattr(config.vision_config, "patch_size", None) is not None @@ -1590,7 +1583,7 @@ def preprocess_inputs( if video is not None and "