diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index 8c1840edc3..28d56061d4 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -39,7 +39,7 @@ jobs: "*quantization*", "*transformation*", ] - transformers-version: ["4.45.0", "4.57.6", "latest"] + transformers-version: ["5.8.0", "latest"] runs-on: ubuntu-22.04 @@ -70,17 +70,7 @@ jobs: run: | uv pip install transformers==${{ matrix.transformers-version }} - - if: ${{ matrix.transformers-version == '4.57.6' }} - name: Install qwen-asr package - run: | - uv pip install qwen-asr - - - if: ${{ matrix.transformers-version == '4.45.0' }} - name: Install specific dependencies and versions required for older transformers - run: | - uv pip install transformers==${{ matrix.transformers-version }} accelerate==0.* peft==0.13.* diffusers==0.32.* transformers_stream_generator - - - if: ${{ matrix.transformers-version != '4.45.0' && matrix.test-pattern == '*decoder*'}} + - if: ${{ matrix.test-pattern == '*decoder*'}} name: Install auto-gptq, autoawq run: | uv pip install auto-gptq "autoawq<0.2.8" diff --git a/.github/workflows/test_openvino_preview_models.yml b/.github/workflows/test_openvino_preview_models.yml index 5643fa56f6..87dcb68abc 100644 --- a/.github/workflows/test_openvino_preview_models.yml +++ b/.github/workflows/test_openvino_preview_models.yml @@ -61,18 +61,8 @@ jobs: run: | uv pip install --pre -U openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - - name: Gemma 4 Validation - run: | - uv pip install transformers==5.5.0 - pytest tests/openvino/${{ matrix.test-pattern }} -k "gemma4 and not gemma4_unified" --durations=0 - - name: Gemma 4 Unified Validation if: ${{ matrix.test-pattern != '*quantization*' }} run: | uv pip install "transformers>=5.10,<5.11" - pytest tests/openvino/${{ matrix.test-pattern }} -k gemma4_unified --durations=0 - - - name: Qwen3.5 and Qwen3.5-Moe Validation - run: | - uv pip install "transformers>=5.2,<5.3" - pytest tests/openvino/${{ matrix.test-pattern }} -k "qwen3_5 or qwen3_5_moe" --durations=0 + pytest tests/openvino/${{ matrix.test-pattern }} -k gemma4_unified --durations=0 \ No newline at end of file diff --git a/.github/workflows/test_openvino_slow.yml b/.github/workflows/test_openvino_slow.yml index 082b211817..5f8bcf83ce 100644 --- a/.github/workflows/test_openvino_slow.yml +++ b/.github/workflows/test_openvino_slow.yml @@ -36,7 +36,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04", "windows-2022"] - transformers-version: ["4.45.0", "latest"] + transformers-version: ["5.8.0", "latest"] include: - transformers-version: "main" os: "ubuntu-22.04" @@ -69,11 +69,6 @@ jobs: path: ~/.cache/huggingface/hub key: hf-models - - if: ${{ matrix.transformers-version != 'latest' && matrix.transformers-version != 'main' }} - name: Install specific dependencies and versions required for older transformers - run: | - uv pip install transformers==${{ matrix.transformers-version }} accelerate==0.* peft==0.13.* diffusers==0.32.* transformers_stream_generator - - if: ${{ matrix.transformers-version == 'main' }} name: Install transformers from repository run: | diff --git a/docs/source/openvino/inference.mdx b/docs/source/openvino/inference.mdx index 6b5d6a3ccf..74accd9c40 100644 --- a/docs/source/openvino/inference.mdx +++ b/docs/source/openvino/inference.mdx @@ -48,7 +48,7 @@ As shown in the table below, each task is associated with a class enabling to au | `OVModelForCausalLM` | `text-generation-with-past` | | `OVModelForSeq2SeqLM` | `text2text-generation-with-past` | | `OVModelForSpeechSeq2Seq` | `automatic-speech-recognition` | -| `OVModelForVision2Seq` | `image-to-text` | +| `OVModelForImageTextToText` | `image-to-text` | | `OVModelForTextToSpeechSeq2Seq` | `text-to-audio` | ### Diffusers models diff --git a/docs/source/openvino/reference.mdx b/docs/source/openvino/reference.mdx index 7547a56b06..790f0f03f7 100644 --- a/docs/source/openvino/reference.mdx +++ b/docs/source/openvino/reference.mdx @@ -102,9 +102,9 @@ The following classes are available for the following computer vision tasks. The following classes are available for the following multimodal tasks. -### OVModelForVision2Seq +### OVModelForImageTextToText -[[autodoc]] openvino.modeling_seq2seq.OVModelForVision2Seq +[[autodoc]] openvino.modeling_seq2seq.OVModelForImageTextToText - forward ### OVModelForPix2Struct diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index d2bfda8db2..063e2a61d2 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -24,7 +24,7 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from requests.exceptions import ConnectionError as RequestsConnectionError -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin +from transformers import AutoConfig, AutoTokenizer, Mxfp4Config, PreTrainedTokenizerBase, ProcessorMixin from transformers.utils import is_torch_available from openvino import Core, Type, save_model @@ -53,9 +53,6 @@ ) -if is_transformers_version(">=", "4.55"): - from transformers import Mxfp4Config - FORCE_ATTN_MODEL_CLASSES = {"phi3_v": "eager", "gemma2": "sdpa", "llama4": "sdpa"} if TYPE_CHECKING: @@ -360,7 +357,7 @@ def main_export( loading_kwargs["config"] = update_config_for_eagle3(config) # mxfp4 quantized model will be dequantized to bf16 - if quant_method == "mxfp4" and is_transformers_version(">=", "4.55"): + if quant_method == "mxfp4": dtype = torch.bfloat16 loading_kwargs["quantization_config"] = Mxfp4Config(dequantize=True) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 204cd573bf..c44ce1e365 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -656,11 +656,7 @@ def export_from_model( ensure_export_task_support_stateful(task) or ensure_model_type_support_stateful(model_type) ) - if ( - stateful - and is_encoder_decoder - and not getattr(model, "_supports_cache_class", is_transformers_version(">=", "4.54")) - ): + if stateful and is_encoder_decoder and not getattr(model, "_supports_cache_class", True): stateful = False # TODO: support ov_config.py in the model repo if custom_architecture and custom_export_configs is None: @@ -1245,7 +1241,9 @@ def get_sd3_models_for_export(pipeline, exporter, int_dtype, float_dtype): text_encoder = getattr(pipeline, "text_encoder", None) if text_encoder is not None: text_encoder.config.output_hidden_states = True - text_encoder.text_model.config.output_hidden_states = True + # `CLIPTextTransformer` removed since transformers v5.6 + if hasattr(text_encoder, "text_model"): + text_encoder.text_model.config.output_hidden_states = True text_encoder_config_constructor = TasksManager.get_exporter_config_constructor( model=text_encoder, exporter=exporter, @@ -1307,7 +1305,9 @@ def get_sd3_models_for_export(pipeline, exporter, int_dtype, float_dtype): text_encoder_2 = getattr(pipeline, "text_encoder_2", None) if text_encoder_2 is not None: text_encoder_2.config.output_hidden_states = True - text_encoder_2.text_model.config.output_hidden_states = True + # `CLIPTextTransformer` removed since transformers v5.6 + if hasattr(text_encoder_2, "text_model"): + text_encoder_2.text_model.config.output_hidden_states = True export_config_constructor = TasksManager.get_exporter_config_constructor( model=text_encoder_2, exporter=exporter, diff --git a/optimum/exporters/openvino/input_generators.py b/optimum/exporters/openvino/input_generators.py index 21500761c9..5176037c21 100644 --- a/optimum/exporters/openvino/input_generators.py +++ b/optimum/exporters/openvino/input_generators.py @@ -30,7 +30,6 @@ FalconDummyPastKeyValuesGenerator, MistralDummyPastKeyValuesGenerator, NormalizedTextConfig, - is_transformers_version, ) from optimum.utils.input_generators import DTYPE_MAPPER from optimum.utils.normalized_config import NormalizedConfig, NormalizedVisionConfig @@ -42,46 +41,27 @@ def __init__(self, task: str, normalized_config: NormalizedTextConfig, **kwargs) self.multi_query = normalized_config.multi_query def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"): - if is_transformers_version("<", "4.54"): - if self.multi_query: - shape = ( - self.batch_size, - self.sequence_length, - self.hidden_size // self.num_attention_heads * 2, - ) - else: - shape = ( - self.batch_size, - self.num_attention_heads, - self.sequence_length, - self.hidden_size // self.num_attention_heads * 2, - ) - pkv = [ - self.random_float_tensor(shape, framework=framework, dtype=float_dtype) for _ in range(self.num_layers) - ] - + if self.multi_query: + shape = ( + self.batch_size, + 1, + self.sequence_length, + self.hidden_size // self.num_attention_heads, + ) else: - if self.multi_query: - shape = ( - self.batch_size, - 1, - self.sequence_length, - self.hidden_size // self.num_attention_heads, - ) - else: - shape = ( - self.batch_size, - self.num_attention_heads, - self.sequence_length, - self.hidden_size // self.num_attention_heads, - ) - pkv = [ - ( - self.random_float_tensor(shape, framework=framework, dtype=float_dtype), - self.random_float_tensor(shape, framework=framework, dtype=float_dtype), - ) - for _ in range(self.num_layers) - ] + shape = ( + self.batch_size, + self.num_attention_heads, + self.sequence_length, + self.hidden_size // self.num_attention_heads, + ) + pkv = [ + ( + self.random_float_tensor(shape, framework=framework, dtype=float_dtype), + self.random_float_tensor(shape, framework=framework, dtype=float_dtype), + ) + for _ in range(self.num_layers) + ] return pkv diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 100f964df0..1f06c2ec7c 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -90,11 +90,8 @@ ArcticModelPatcher, BaichuanModelPatcher, BigBirdPegasusModelPatcher, - BlenderbotModelPatcher, - BlenderbotSmallModelPatcher, BloomModelPatcher, ChatGLMModelPatcher, - CLIPModelPatcher, CodeGenModelPatcher, CommonImageEmbeddingsModelPatcher, DBRXModelPatcher, @@ -112,7 +109,6 @@ Gemma4UnifiedLMModelPatcher, GptJModelPatcher, GptNeoModelPatcher, - GptNeoxModelPatcher, GptOssModelPatcher, GraniteMoeHybridModelPatcher, GraniteMoEModelPatcher, @@ -134,7 +130,6 @@ LlavaQwen2ImageEmbeddingsModelPatcher, MairaImageEmbeddingModelPatcher, MambaPatcher, - MarianModelPatcher, MiniCPM3Patcher, MiniCPMModelPatcher, MiniCPMVImageEmbeddingsModelPatcher, @@ -145,8 +140,6 @@ MPTModelPatcher, OVDecoderModelPatcher, OVSeq2SeqModelPatcher, - PegasusModelPatcher, - PersimmonModelPatcher, Phi3ModelPatcher, Phi3VisionImageEmbeddingsPatcher, Phi4MMAudioEncoderPatcher, @@ -284,38 +277,6 @@ def init_model_configs(): "AutoModelForCausalLM", ) - # since transformers v4.46, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.46.0/src/transformers/models/auto/modeling_auto.py#L776 - if is_transformers_version("<", "4.46"): - TasksManager._CUSTOM_CLASSES[("pt", "llava", "image-text-to-text")] = ( - "transformers", - "LlavaForConditionalGeneration", - ) - TasksManager._CUSTOM_CLASSES[("pt", "llava_next", "image-text-to-text")] = ( - "transformers", - "LlavaNextForConditionalGeneration", - ) - TasksManager._CUSTOM_CLASSES[("pt", "qwen2_vl", "image-text-to-text")] = ( - "transformers", - "Qwen2VLForConditionalGeneration", - ) - - # since transformers v4.50, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.50.0/src/transformers/models/auto/modeling_auto.py#L835 - if is_transformers_version("<", "4.50"): - TasksManager._CUSTOM_CLASSES[("pt", "gemma3", "image-text-to-text")] = ( - "transformers", - "Gemma3ForConditionalGeneration", - ) - - # since transformers v4.52, model can be loaded using default AutoModelForImageTextToText - # https://github.com/huggingface/transformers/blob/v4.52.0/src/transformers/models/auto/modeling_auto.py#L899 - if is_transformers_version("<", "4.52"): - TasksManager._CUSTOM_CLASSES[("pt", "llava_next_video", "image-text-to-text")] = ( - "transformers", - "AutoModelForVision2Seq", - ) - # Qwen3-ASR is loaded via trust_remote_code; register custom classes for task lookup. if is_transformers_version("==", "4.57.6"): TasksManager._CUSTOM_CLASSES[("pt", "qwen3_asr", "automatic-speech-recognition")] = ( @@ -764,7 +725,6 @@ class ExaoneOpenVINOConfig(LlamaOpenVINOConfig): ) class Exaone4OpenVINOConfig(LlamaOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.54.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -1017,13 +977,11 @@ def inputs(self) -> Dict[str, Dict[int, str]]: ) class PersimmonOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = PersimmonModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager("biogpt", *["text-generation", "text-generation-with-past"], library_name="transformers") -class BioGPTOpenVINOConfig( - TextDecoderWithPositionIdsOpenVINOConfig if is_transformers_version(">=", "4.52.0") else TextDecoderOpenVINOConfig -): +class BioGPTOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig _MODEL_PATCHER = OVDecoderModelPatcher @@ -1080,24 +1038,6 @@ class BloomOpenVINOConfig(TextDecoderOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig.with_args(num_layers="n_layer", num_attention_heads="n_head") _MODEL_PATCHER = BloomModelPatcher - def add_past_key_values(self, inputs_or_outputs: Dict[str, Dict[int, str]], direction: str): - if is_transformers_version(">=", "4.44"): - super().add_past_key_values(inputs_or_outputs, direction) - else: - if direction not in ["inputs", "outputs"]: - raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') - - if direction == "inputs": - decoder_sequence_name = "past_sequence_length" - name = "past_key_values" - else: - decoder_sequence_name = "past_sequence_length + sequence_length" - name = "present" - - for i in range(self._normalized_config.num_layers): - inputs_or_outputs[f"{name}.{i}.key"] = {0: "batch_size * num_heads", 2: decoder_sequence_name} - inputs_or_outputs[f"{name}.{i}.value"] = {0: "batch_size * num_heads", 1: decoder_sequence_name} - @register_in_tasks_manager( "cohere", @@ -1234,7 +1174,7 @@ class MistralOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): ) class GPTNeoxOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = GptNeoxModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager( @@ -1243,7 +1183,7 @@ class GPTNeoxOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): class GPTNeoxJapaneseOpenVINOConfig(TextDecoderOpenVINOConfig): # GPTNeoxJapanese does not require position_ids input. NORMALIZED_CONFIG_CLASS = NormalizedTextConfig - _MODEL_PATCHER = GptNeoxModelPatcher + _MODEL_PATCHER = OVDecoderModelPatcher @register_in_tasks_manager( @@ -1371,7 +1311,6 @@ class SiglipNormalizedConfig(CLIPNormalizedConfig): @register_in_tasks_manager("clip", *["zero-shot-image-classification"], library_name="open_clip") class OpenCLIPOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = CLIPNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1419,7 +1358,6 @@ class OpenCLIPTextOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1474,7 +1412,6 @@ def rename_ambiguous_inputs(self, inputs): ) class CLIPOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = CLIPNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1512,7 +1449,6 @@ class CLIPTextOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1542,7 +1478,6 @@ class CLIPTextWithProjectionOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1566,7 +1501,6 @@ def outputs(self) -> Dict[str, Dict[int, str]]: @register_in_tasks_manager("clip_vision_model", *["feature-extraction"], library_name="transformers") class CLIPVisionModelOpenVINOConfig(VisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> Dict[str, Dict[int, str]]: @@ -1661,10 +1595,7 @@ def generate_dummy_inputs(self, framework: str = "pt", **kwargs): ) dummy_inputs["inputs_embeds"] = inputs_embeds if "token_type_ids" in self.inputs: - if is_transformers_version(">=", "4.53"): - token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1] + pask_key_values[0][0].shape[-2]) - else: - token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1]) + token_type_ids_shape = (input_ids.shape[0], input_ids.shape[1] + pask_key_values[0][0].shape[-2]) dummy_inputs["token_type_ids"] = self.orig_export_config.DUMMY_INPUT_GENERATOR_CLASSES[ 0 ].random_int_tensor(token_type_ids_shape, min_value=0, max_value=2) @@ -1927,7 +1858,6 @@ class LlavaNextVideoConfigBehavior(str, enum.Enum): @register_in_tasks_manager("llava_next_video", *["image-text-to-text"], library_name="transformers") class LlavaNextVideoOpenVINOConfig(LlavaOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.42.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" SUPPORTED_BEHAVIORS = [model_type.value for model_type in LlavaNextVideoConfigBehavior] @@ -3486,12 +3416,6 @@ def inputs(self) -> Dict[str, Dict[int, str]]: # TODO: this can be fixed by generating the correct inputs in the input generator common_inputs["encoder_outputs"] = {0: "batch_size", 1: "encoder_sequence_length"} - if self._behavior in {ConfigBehavior.DECODER, ConfigBehavior.MONOLITH}: - if is_transformers_version(">=", "4.43.0") and is_transformers_version("<", "4.46.0"): - # since https://github.com/huggingface/transformers/pull/31166 - if self._behavior is not ConfigBehavior.ENCODER and self.use_past_in_inputs: - common_inputs["cache_position"] = {0: "decoder_sequence_length"} - return common_inputs @property @@ -3625,7 +3549,6 @@ class T5OpenVINOConfig(TextSeq2SeqOpenVINOConfig): library_name="transformers", ) class MT5OpenVINOConfig(T5OpenVINOConfig): - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -3830,7 +3753,6 @@ class DeepseekOpenVINOConfig(MiniCPM3OpenVINOConfig): @register_in_tasks_manager("got_ocr2", *["image-to-text", "image-text-to-text"], library_name="transformers") class GotOCR2OpenVINOConfig(BaseVLMOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.49.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" def __init__( @@ -3923,6 +3845,7 @@ class Gemma4ConfigBehavior(str, enum.Enum): @register_in_tasks_manager("gemma4", *["image-text-to-text"], library_name="transformers") class Gemma4OpenVINOConfig(Gemma3OpenVINOConfig): + MIN_TRANSFORMERS_VERSION = "5.5" SUPPORTED_BEHAVIORS = [model_type.value for model_type in Gemma4ConfigBehavior] DUMMY_INPUT_GENERATOR_CLASSES = (DummyVisionInputGenerator, DummyTextInputGenerator) @@ -4122,6 +4045,8 @@ def outputs(self) -> Dict[str, Dict[int, str]]: @register_in_tasks_manager("gemma3n", *["image-text-to-text"], library_name="transformers") class Gemma3nOpenVINOConfig(Gemma4OpenVINOConfig): + MIN_TRANSFORMERS_VERSION = "5.0" + def __init__( self, config: "PretrainedConfig", @@ -4353,7 +4278,6 @@ def inputs(self) -> Dict[str, Dict[int, str]]: class Idefics3OpenVINOConfig(BaseVLMOpenVINOConfig): DUMMY_INPUT_GENERATOR_CLASSES = (DummyVisionInputGenerator, DummyVisionPositionIdsInputGenerator) MIN_TRANSFORMERS_VERSION = "4.46.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" def __init__( @@ -4413,7 +4337,6 @@ def get_model_for_behavior(self, model, behavior: Union[str, VLMConfigBehavior]) @register_in_tasks_manager("smolvlm", *["image-text-to-text"], library_name="transformers") class SmolVLMOpenVINOConfig(Idefics3OpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.50.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -4430,7 +4353,7 @@ class SmolVLMOpenVINOConfig(Idefics3OpenVINOConfig): library_name="transformers", ) class BlenderbotOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = BlenderbotModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher @register_in_tasks_manager( @@ -4446,7 +4369,7 @@ class BlenderbotOpenVINOConfig(BartOpenVINOConfig): library_name="transformers", ) class BlenderbotSmallOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = BlenderbotSmallModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher @register_in_tasks_manager( @@ -4462,7 +4385,7 @@ class BlenderbotSmallOpenVINOConfig(BartOpenVINOConfig): library_name="transformers", ) class PegasusOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = PegasusModelPatcher + _MODEL_PATCHER = OVSeq2SeqModelPatcher def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -4482,8 +4405,7 @@ def __init__(self, *args, **kwargs): library_name="transformers", ) class MarianOpenVINOConfig(BartOpenVINOConfig): - _MODEL_PATCHER = MarianModelPatcher - # TODO (@echarlaix): add v5 support + _MODEL_PATCHER = OVSeq2SeqModelPatcher MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -4663,7 +4585,6 @@ def overwrite_shape_and_generate_input( ) class Llama4TextOpenVINOConfig(LlamaOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.51.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" DUMMY_INPUT_GENERATOR_CLASSES = (DummyTextInputGenerator, GemmaDummyPastKeyValuesGenerator) DUMMY_PKV_GENERATOR_CLASS = GemmaDummyPastKeyValuesGenerator @@ -4675,7 +4596,6 @@ class Llama4TextOpenVINOConfig(LlamaOpenVINOConfig): ) class Llama4OpenVINOConfig(GotOCR2OpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.51.0" - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" def __init__(self, *args, **kwargs): @@ -4698,6 +4618,7 @@ class MambaOpenVINOConfig(TextDecoderOpenVINOConfig): DUMMY_PKV_GENERATOR_CLASS = MambaCacheDummyInputGenerator NORMALIZED_CONFIG_CLASS = NormalizedTextConfig MIN_TRANSFORMERS_VERSION = "4.43.0" + MAX_TRANSFORMERS_VERSION = "5.3.0" _MODEL_PATCHER = MambaPatcher @property @@ -4878,8 +4799,8 @@ def inputs(self) -> Dict[str, Dict[int, str]]: ) class LFM2OpenVINOConfig(MambaOpenVINOConfig): MIN_TRANSFORMERS_VERSION = "4.54.0" + MAX_TRANSFORMERS_VERSION = "5.4.0" _MODEL_PATCHER = Lfm2ModelPatcher - DUMMY_INPUT_GENERATOR_CLASSES = (DummyTextInputGenerator, Lfm2DummyPastKeyValuesGenerator) DUMMY_PKV_GENERATOR_CLASS = Lfm2DummyPastKeyValuesGenerator @@ -5003,9 +4924,7 @@ class Olmo2OOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): @register_in_tasks_manager("opt", *[*COMMON_TEXT_GENERATION_TASKS, "text-classification", "question-answering"]) -class OPTOpenVINOConfig( - TextDecoderWithPositionIdsOpenVINOConfig if is_transformers_version(">=", "4.46.0") else TextDecoderOpenVINOConfig -): +class OPTOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedTextConfig def __init__(self, *args, **kwargs): @@ -5021,37 +4940,6 @@ class GPTBigCodeOpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): NORMALIZED_CONFIG_CLASS = NormalizedConfigManager.get_normalized_config_class("gpt_bigcode") DUMMY_PKV_GENERATOR_CLASS = GPTBigCodeDummyPastKeyValuesGenerator - def add_past_key_values(self, inputs_or_outputs: dict, direction: str): - if is_transformers_version(">=", "4.54"): - super().add_past_key_values(inputs_or_outputs, direction) - else: - if direction not in ["inputs", "outputs"]: - raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given') - - if direction == "inputs": - decoder_sequence_name = "past_sequence_length" - name = "past_key_values" - else: - decoder_sequence_name = "past_sequence_length + sequence_length" - name = "present" - - if self._normalized_config.multi_query: - decoder_sequence_dim = 1 - else: - decoder_sequence_dim = 2 - - for i in range(self._normalized_config.num_layers): - inputs_or_outputs[f"{name}.{i}.key_value"] = { - 0: "batch_size", - decoder_sequence_dim: decoder_sequence_name, - } - - def flatten_past_key_values(self, flattened_output, name, idx, t): - if is_transformers_version(">=", "4.54"): - super().flatten_past_key_values(flattened_output, name, idx, t) - else: - flattened_output[f"{name}.{idx}.key_value"] = t - class _Pix2StructNormalizedConfig(NormalizedSeq2SeqConfig): ENCODER_NUM_LAYERS = "vision_config.num_hidden_layers" @@ -5082,12 +4970,6 @@ class Pix2StructOpenVINOConfig(OpenVINOSeq2SeqConfigWithPast): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if is_transformers_version("==", "4.46.0"): - logger.warning( - "Found transformers v4.46.0 while trying to export a Pix2Struct model, " - "this specific version of transformers is broken for this model. Please " - "upgrade to v4.46.1 or higher, or downgrade to v4.45.x.", - ) @property def inputs(self): @@ -5186,7 +5068,7 @@ class AlbertOpenVINOConfig(BertOpenVINOConfig): @register_in_tasks_manager("nystromformer", *COMMON_TEXT_TASKS) class NystromformerOpenVINOConfig(BertOpenVINOConfig): - pass + MAX_TRANSFORMERS_VERSION = "4.50.3" @register_in_tasks_manager("convbert", *COMMON_TEXT_TASKS) @@ -5216,7 +5098,6 @@ class MobileBertOpenVINOConfig(BertOpenVINOConfig): @register_in_tasks_manager("xlm", *COMMON_TEXT_TASKS) class XLMOpenVINOConfig(BertOpenVINOConfig): - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -5250,7 +5131,6 @@ class CamembertOpenVINOConfig(DistilBertOpenVINOConfig): @register_in_tasks_manager("flaubert", *COMMON_TEXT_TASKS) class FlaubertOpenVINOConfig(BertOpenVINOConfig): - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -5307,7 +5187,6 @@ class Data2VecAudioOpenVINOConfig(HubertOpenVINOConfig): @register_in_tasks_manager("data2vec-text", *COMMON_TEXT_TASKS) class Data2VecTextOpenVINOConfig(DistilBertOpenVINOConfig): - # TODO (@echarlaix): add v5 support MAX_TRANSFORMERS_VERSION = "4.57.6" @@ -5627,7 +5506,6 @@ def outputs(self) -> dict: @register_in_tasks_manager("siglip", *["feature-extraction", "zero-shot-image-classification"]) class SiglipOpenVINOConfig(TextAndVisionOpenVINOConfig): NORMALIZED_CONFIG_CLASS = SiglipNormalizedConfig - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> dict: @@ -5685,7 +5563,6 @@ class SiglipTextWithProjectionOpenVINOConfig(TextEncoderOpenVINOConfig): num_layers="num_hidden_layers", allow_new=True, ) - _MODEL_PATCHER = CLIPModelPatcher @property def inputs(self) -> dict: @@ -5708,8 +5585,6 @@ def outputs(self) -> dict: @register_in_tasks_manager("siglip-text", *["feature-extraction"]) class SiglipTextOpenVINOConfig(SiglipTextWithProjectionOpenVINOConfig): - _MODEL_PATCHER = CLIPModelPatcher - @property def outputs(self) -> dict: common_outputs = { @@ -5961,6 +5836,7 @@ def generate_dummy_inputs(self, framework: str = "pt", **kwargs): ) class LFM2MoeOpenVINOConfig(LFM2OpenVINOConfig): MIN_TRANSFORMERS_VERSION = "5.0" + MAX_TRANSFORMERS_VERSION = "5.4.0" _MODEL_PATCHER = Lfm2MoeModelPatcher diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index bf1706cd0a..50c371f421 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -15,7 +15,6 @@ import functools import inspect import logging -import logging as log import math import types from dataclasses import dataclass @@ -29,6 +28,12 @@ from transformers.cache_utils import Cache, DynamicCache, EncoderDecoderCache from transformers.configuration_utils import PretrainedConfig from transformers.generation import GenerationMixin +from transformers.masking_utils import ( + ALL_MASK_ATTENTION_FUNCTIONS, + create_causal_mask, + eager_mask, + sdpa_mask, +) from transformers.modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPast, @@ -43,9 +48,10 @@ LlamaRotaryEmbedding, ) from transformers.models.phi3.modeling_phi3 import apply_rotary_pos_emb, repeat_kv +from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock from transformers.models.speecht5.modeling_speecht5 import SpeechT5EncoderWithSpeechPrenet from transformers.processing_utils import Unpack -from transformers.utils import ModelOutput +from transformers.utils import ModelOutput, TransformersKwargs from optimum.exporters.openvino._ov_ops import convert_recurrent_attention_cell from optimum.exporters.openvino.base import OpenVINOConfig @@ -64,28 +70,6 @@ ) -if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - from transformers.models.clip.modeling_clip import CLIPAttention, CLIPSdpaAttention - - -if is_transformers_version(">=", "4.48"): - from transformers.cache_utils import DynamicCache, EncoderDecoderCache -if is_transformers_version(">=", "4.53"): - from transformers.masking_utils import ( - ALL_MASK_ATTENTION_FUNCTIONS, - eager_mask, - sdpa_mask, - ) - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock - - -if is_transformers_version(">=", "4.54"): - from transformers.masking_utils import create_causal_mask - from transformers.utils import TransformersKwargs -else: - TransformersKwargs = object - - if is_transformers_version(">=", "4.56"): import transformers.masking_utils @@ -206,28 +190,6 @@ def patched_forward(input_ids, attention_mask): self.patched_forward = patched_forward -class CLIPModelPatcher(ModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - self.original_sdpa_forward = CLIPSdpaAttention.forward - CLIPSdpaAttention.forward = CLIPAttention.forward - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.43") and is_transformers_version("<", "4.48"): - CLIPSdpaAttention.forward = self.original_sdpa_forward - - -# This is a traceable version of the original function, -# the original results in a constant integer due to the use of int(expr) -def _get_feat_extract_output_lengths_patched(self, input_lengths: torch.LongTensor): - output_conv1_length = (input_lengths - 127) // 64 + 1 - output_conv2_length = (output_conv1_length - 7) // 3 + 1 - output_conv3_length = (output_conv2_length - 3) // 2 + 1 - return output_conv3_length - - def _get_model_attribute(model, name): target = getattr(model, "model", model) if is_transformers_version(">=", "5") else model return getattr(target, name) @@ -540,19 +502,14 @@ def __enter__(self): if hasattr(self._model, "model"): patch_cos_sin_cached_fp32(self._model.model) - if is_transformers_version("<", "4.53.0") and hasattr(self._model, "_update_causal_mask"): - self._model._update_causal_mask_original = self._model._update_causal_mask - self._model._update_causal_mask = types.MethodType(_update_causal_mask_patched, self._model) + # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min + # Although I'm not sure this is the right way to handle this, we are basically pretending that -65,504 is -inf + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) - if is_transformers_version(">=", "4.53.0"): - # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min - # Although I'm not sure this is the right way to handle this, we are basically pretending that -65,504 is -inf - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) - - # for decoder models, we use eager mask without vmap for sdpa as well - # to avoid a nan output issue in OpenVINO that only happens in case of: - # non-stateful models on cpu and stateful models on npu - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # for decoder models, we use eager mask without vmap for sdpa as well + # to avoid a nan output issue in OpenVINO that only happens in case of: + # non-stateful models on cpu and stateful models on npu + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) for module in self._model.modules(): rope_type = getattr(module, "rope_type", None) @@ -566,13 +523,8 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53") and hasattr(self._model, "_update_causal_mask_original"): - self._model._update_causal_mask = self._model._update_causal_mask_original - del self._model._update_causal_mask_original - - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) for module in self._model.modules(): if hasattr(module, "_rope_orig_forward"): @@ -849,124 +801,9 @@ def create_embed_positions_buffer(rotary_emb, max_position_embeddings: int = Non return create_sinusoidal_positions(max_position_embeddings, dim, base, inv_freq) -# copied from https://github.com/huggingface/transformers/commit/57d7594a79a9f5d835abf2d4d384db0e4818e548 to unblock export with transformers 4.42 -def _mistral_update_causal_mask( - self, - attention_mask: torch.Tensor, - input_tensor: torch.Tensor, - cache_position: torch.Tensor, - past_key_values: "Cache", - use_cache: bool, - output_attentions: bool, -): - from transformers.cache_utils import SlidingWindowCache, StaticCache - from transformers.modeling_attn_mask_utils import AttentionMaskConverter - - # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static - # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. - # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using - # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 - - if self._attn_implementation == "flash_attention_2": - if attention_mask is not None and use_cache: - is_padding_right = attention_mask[:, -1].sum().item() != input_tensor.size()[0] - if is_padding_right: - raise ValueError( - "You are attempting to perform batched generation with padding_side='right'" - " this may lead to unexpected behaviour for Flash Attention version of Mistral. Make sure to " - " call `tokenizer.padding_side = 'left'` before tokenizing the input. " - ) - if attention_mask is not None and 0.0 in attention_mask: - return attention_mask - return None - - # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in - # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail - # to infer the attention mask. - - # cache_position must be valid here no matter which cache we use - past_seen_tokens = cache_position[0] if past_key_values is not None else 0 - using_static_cache = isinstance(past_key_values, StaticCache) - using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache) - - if ( - self.config._attn_implementation == "sdpa" - and not (using_static_cache or using_sliding_window_cache) - and not output_attentions - ): - if AttentionMaskConverter._ignore_causal_mask_sdpa( - attention_mask, - inputs_embeds=input_tensor, - past_key_values_length=past_seen_tokens, - sliding_window=self.config.sliding_window, - is_training=self.training, - ): - return None - - dtype, device = input_tensor.dtype, input_tensor.device - min_dtype = torch.finfo(torch.float16).min - sequence_length = input_tensor.shape[1] - # SlidingWindowCache - if using_sliding_window_cache: - target_length = max(sequence_length, self.config.sliding_window) - # StaticCache - elif using_static_cache: - target_length = past_key_values.get_max_length() - # DynamicCache or no cache - else: - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else past_seen_tokens + sequence_length + 1 - ) - - if attention_mask is not None and attention_mask.dim() == 4: - # in this case we assume that the mask comes already in inverted form and requires no inversion or slicing - if attention_mask.max() != 0: - raise ValueError("Custom 4D attention mask should be passed in inverted form with max==0`") - causal_mask = attention_mask - else: - causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) - exclude_mask = torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) - if self.config.sliding_window is not None: - if not using_sliding_window_cache or sequence_length > self.config.sliding_window: - exclude_mask = exclude_mask.bitwise_or( - torch.arange(target_length, device=device) - <= (cache_position.reshape(-1, 1) - self.config.sliding_window) - ) - causal_mask *= exclude_mask - causal_mask = causal_mask[None, None, :, :].expand(input_tensor.shape[0], 1, -1, -1) - if attention_mask is not None: - causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit - if attention_mask.dim() == 2: - mask_length = attention_mask.shape[-1] - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] - padding_mask = padding_mask == 0 - causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( - padding_mask, min_dtype - ) - - if ( - self.config._attn_implementation == "sdpa" - and attention_mask is not None - and attention_mask.device.type == "cuda" - and not output_attentions - ): - # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when - # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. - # Details: https://github.com/pytorch/pytorch/issues/110213 - causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) - - return causal_mask - - class MistralModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version("<", "4.48.0"): - # apply fix https://github.com/huggingface/transformers/commit/57d7594a79a9f5d835abf2d4d384db0e4818e548 - self._model.model._orig_update_causal_mask = self._model.model._update_causal_mask - self._model.model._update_causal_mask = types.MethodType(_mistral_update_causal_mask, self._model.model) if hasattr(self._model, "model") and hasattr(self._model.model, "layers"): for layer in self._model.model.layers: @@ -984,10 +821,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.48.0"): - self._model.model._update_causal_mask = self._model.model._orig_update_causal_mask - del self._model.model._orig_update_causal_mask - if hasattr(self._model.model, "model") and hasattr(self._model.model.model, "layers"): for layer in self._model.model.layers: if hasattr(layer.self_attn, "rotary_emb"): @@ -1408,17 +1241,9 @@ def _mpt_sdpa_attention_forward( if past_key_value is not None: # starting from v4.54 https://github.com/huggingface/transformers/blob/v4.54.0/src/transformers/models/mpt/modeling_mpt.py#L362 - if is_transformers_version(">=", "4.54"): - cache_kwargs = {"cache_position": cache_position} - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - pkv_seq_length = past_key_value.get_seq_length() - - else: - if len(past_key_value) != 0: - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - past_key_value = (key_states, value_states) - pkv_seq_length = past_key_value[0].shape[2] + cache_kwargs = {"cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + pkv_seq_length = past_key_value.get_seq_length() key_length = key_states.shape[-2] query_length = seq_length if past_key_value is None else seq_length + pkv_seq_length @@ -1447,9 +1272,6 @@ def _mpt_sdpa_attention_forward( outputs = (attn_output, None) - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - return outputs @@ -1499,9 +1321,6 @@ def _mpt_block_forward( output = self.ffn(layernorm_output, residual) outputs = (output,) - if use_cache and is_transformers_version("<", "4.54"): - outputs += (attn_out[2],) - if output_attentions: outputs += (attn_out[1],) @@ -1645,133 +1464,6 @@ def __exit__(self, exc_type, exc_value, traceback): block.attention.forward = block.attention._orig_forward -def phi3_442_forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - **kwargs, -) -> Union[Tuple, BaseModelOutputWithPast]: - from transformers.cache_utils import Cache - from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask - - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - use_cache = use_cache if use_cache is not None else self.config.use_cache - - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - # retrieve input_ids and inputs_embeds - if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif input_ids is not None: - batch_size, seq_length = input_ids.shape[:2] - elif inputs_embeds is not None: - batch_size, seq_length = inputs_embeds.shape[:2] - else: - raise ValueError("You have to specify either input_ids or inputs_embeds") - - past_key_values_length = 0 - - if use_cache: - use_legacy_cache = not isinstance(past_key_values, Cache) - if use_legacy_cache: - if is_transformers_version("<", "5"): - past_key_values = DynamicCache.from_legacy_cache(past_key_values) - else: - past_key_values = DynamicCache(past_key_values) - - past_key_values_length = past_key_values.get_usable_length(seq_length) - - if position_ids is None: - device = input_ids.device if input_ids is not None else inputs_embeds.device - position_ids = torch.arange( - past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device - ) - position_ids = position_ids.unsqueeze(0).view(-1, seq_length) - else: - position_ids = position_ids.view(-1, seq_length).long() - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) - - if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache: - is_padding_right = attention_mask[:, -1].sum().item() != batch_size - if is_padding_right: - raise ValueError( - "You are attempting to perform batched generation with padding_side='right'" - " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to " - " call `tokenizer.padding_side = 'left'` before tokenizing the input. " - ) - - if self._attn_implementation == "flash_attention_2": - # 2d mask is passed through the layers - attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None - else: - # 4d mask is passed through the layers - attention_mask = _prepare_4d_causal_attention_mask( - attention_mask, - (batch_size, seq_length), - inputs_embeds, - past_key_values_length, - sliding_window=self.config.sliding_window, - ) - - hidden_states = inputs_embeds - - # decoder layers - all_hidden_states = () if output_hidden_states else None - all_self_attns = () if output_attentions else None - next_decoder_cache = None - - for decoder_layer in self.layers: - if output_hidden_states: - all_hidden_states += (hidden_states,) - else: - layer_outputs = decoder_layer( - hidden_states, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_value=past_key_values, - output_attentions=output_attentions, - use_cache=use_cache, - ) - - hidden_states = layer_outputs[0] - - if use_cache: - next_decoder_cache = layer_outputs[2 if output_attentions else 1] - - if output_attentions: - all_self_attns += (layer_outputs[1],) - - hidden_states = self.norm(hidden_states) - - # add hidden states from the last decoder layer - if output_hidden_states: - all_hidden_states += (hidden_states,) - - next_cache = None - if use_cache: - next_cache = postprocess_past_key_values(next_decoder_cache) if use_legacy_cache else next_decoder_cache - if not return_dict: - return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) - return BaseModelOutputWithPast( - last_hidden_state=hidden_states, - past_key_values=next_cache, - hidden_states=all_hidden_states, - attentions=all_self_attns, - ) - - # Adapted from https://github.com/huggingface/transformers/blob/ccdabc5642bf84849af93f591e207dc625c8e1e1/src/transformers/models/phi3/modeling_phi3.py#L729 def _phi3_self_attn_sdpa_forward( self, @@ -1854,19 +1546,11 @@ def __enter__(self): # LongRoPE is handled via _longrope_forward in OVDecoderModelPatcher for both transformers >= 5 and < 5, # so keep the original max_position_embeddings to preserve the correct attention scaling factor. - if is_transformers_version("<", "4.48.0"): - self._model.model._orig_forward = self._model.model.forward - self._model.model.forward = types.MethodType(phi3_442_forward, self._model.model) - # https://github.com/huggingface/transformers/blob/30ee508c6c92a1c0aa0281d193c7c0fb815b8d2f/src/transformers/models/phi3/modeling_phi3.py#L113 # init inv_freq for torchscript tracing # 4.48 transformers version phi3 fixed, but issue still visible with trust_remote_true=True (trust_remote_code has _support_sdpa = False) for layer in self._model.model.layers: - if ( - is_torch_version(">=", "2.1.0") - and is_transformers_version("<", "4.48.0") - or not getattr(self._model, "_supports_sdpa", False) - ): + if not getattr(self._model, "_supports_sdpa", False): orig_self_attn_fwd = layer.self_attn.forward layer.self_attn.forward = types.MethodType(_phi3_self_attn_sdpa_forward, layer.self_attn) layer.self_attn._orig_forward = orig_self_attn_fwd @@ -2376,10 +2060,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask_original"): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - for layer in self._model.transformer.h: if hasattr(layer.attn, "_orig_attn"): layer.attn._attn = layer.attn._orig_attn @@ -2564,126 +2244,6 @@ def __exit__(self, exc_type, exc_value, traceback): block.norm_attn_norm.attn.rotary_emb.forward = block.norm_attn_norm.attn.rotary_emb._orig_forward -# Adapted from https://github.com/huggingface/transformers/blob/v4.41.0/src/transformers/models/persimmon/modeling_persimmon.py#L264 -def _persimmon_self_attn_sdpa_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_value: Optional["Cache"] = None, - output_attentions: bool = False, - use_cache: bool = False, - cache_position: Optional[torch.LongTensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - from transformers.models.persimmon.modeling_persimmon import apply_rotary_pos_emb - - if output_attentions: - return self._orig_forward( - hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache - ) - - bsz, q_len, _ = hidden_states.size() - - # [batch_size, seq_length, 3 x hidden_size] - fused_qkv = self.query_key_value(hidden_states) - - # 3 x [batch_size, seq_length, num_heads, head_dim] - (query_states, key_states, value_states) = self._split_heads(fused_qkv) - - if self.qk_layernorm: - query_states = self.q_layernorm(query_states) - key_states = self.k_layernorm(key_states) - - # [batch_size, num_heads, seq_length, head_dim] -> [batch_size, seq_length, num_heads, head_dim] - query_states = query_states.transpose(1, 2) - value_states = value_states.transpose(1, 2) - key_states = key_states.transpose(1, 2) - - if position_embeddings is None: - log.warning( - "The attention layers in this model are transitioning from computing the RoPE embeddings internally " - "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " - "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " - "removed and `position_embeddings` will be mandatory." - ) - cos, sin = self.rotary_emb(value_states, position_ids) - else: - cos, sin = position_embeddings - - rotary_ndims = self.rotary_ndims - # Partial rotary embedding - query_rot, query_pass = ( - query_states[..., :rotary_ndims], - query_states[..., rotary_ndims:], - ) - key_rot, key_pass = ( - key_states[..., :rotary_ndims], - key_states[..., rotary_ndims:], - ) - # [batch_size, seq_length, num_heads, head_dim // config.partial_rotary_factor] - query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids) - - # [batch_size, seq_length, num_heads, head_dim] - query_states = torch.cat((query_rot, query_pass), dim=-1) - key_states = torch.cat((key_rot, key_pass), dim=-1) - - if past_key_value is not None: - # Specific to RoPE models with partial rotation - cache_kwargs = { - "sin": sin, - "cos": cos, - "partial_rotation_size": rotary_ndims, - "cache_position": cache_position, - } - key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) - - causal_mask = attention_mask - if attention_mask is not None: # no matter the length, we just slice it - causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] - - attn_output = F.scaled_dot_product_attention( - query_states, - key_states, - value_states, - causal_mask, - scale=1 / math.sqrt(self.head_dim), - dropout_p=self.attention_dropout.p, - ) - - attn_output = attn_output.transpose(1, 2).contiguous() - attn_output = attn_output.reshape(bsz, q_len, self.hidden_size) - - attn_output = self.dense(attn_output) - - outputs = (attn_output, None) - - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - - return outputs - - -class PersimmonModelPatcher(OVDecoderModelPatcher): - def __enter__(self): - super().__enter__() - - if is_transformers_version("<", "4.56"): - for layer in self._model.model.layers: - if is_torch_version(">=", "2.1.0"): - orig_self_attn_fwd = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(_persimmon_self_attn_sdpa_forward, layer.self_attn) - layer.self_attn._orig_forward = orig_self_attn_fwd - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - - if is_transformers_version("<", "4.56"): - for layer in self._model.model.layers: - if hasattr(layer.self_attn, "_orig_forward"): - layer.self_attn.forward = layer.self_attn._orig_forward - - def _jais_attn_forward( self, hidden_states: Optional[Tuple[torch.FloatTensor]], @@ -2805,178 +2365,14 @@ def __exit__(self, exc_type, exc_value, traceback): layer.attn.forward = layer.attn._orig_forward -# Adapted from https://github.com/huggingface/transformers/blob/31f9a289a6207be6cae746e009d8e0db523be203/src/transformers/models/falcon/modeling_falcon.py#L1138 -def _falcon_prepare_4d_causal_attention_mask_with_cache_position( - attention_mask: torch.Tensor, - sequence_length: int, - target_length: int, - dtype: torch.dtype, - device: torch.device, - cache_position: torch.Tensor, - batch_size: int, - **kwargs, -): - if attention_mask is not None and attention_mask.dim() == 4: - # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing. - causal_mask = attention_mask - else: - # different from original: allow to provide min_dtype as parameter - min_dtype = torch.finfo(dtype).min if "min_dtype" not in kwargs else kwargs["min_dtype"] - causal_mask = torch.full((sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device) - if sequence_length != 1: - causal_mask = torch.triu(causal_mask, diagonal=1) - causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) - causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) - if attention_mask is not None: - causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit - mask_length = attention_mask.shape[-1] - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] - padding_mask = padding_mask == 0 - causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( - padding_mask, min_dtype - ) - - return causal_mask - - -def _falcon_update_causal_mask( - self, - attention_mask: torch.Tensor, - input_tensor: torch.Tensor, - cache_position: torch.Tensor, - past_key_values: "Cache", - output_attentions: bool, - head_mask: torch.Tensor, - alibi: torch.Tensor, -): - # copied from https://github.com/huggingface/transformers/blob/a30c865f991dfec9452cc64bd9a97bfbb96be036/src/transformers/models/falcon/modeling_falcon.py#L1130 - from transformers.cache_utils import StaticCache - from transformers.modeling_attn_mask_utils import AttentionMaskConverter - - # TODO: As of torch==2.2.0, the `attention_mask` passed to the model in `generate` is 2D and of dynamic length even when the static - # KV cache is used. This is an issue for torch.compile which then recaptures cudagraphs at each decode steps due to the dynamic shapes. - # (`recording cudagraph tree for symint key 13`, etc.), which is VERY slow. A workaround is `@torch.compiler.disable`, but this prevents using - # `fullgraph=True`. See more context in https://github.com/huggingface/transformers/pull/29114 - - if self.config._attn_implementation == "flash_attention_2": - if attention_mask is not None and 0.0 in attention_mask: - return attention_mask - return None - - # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in - # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail - # to infer the attention mask. - past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 - using_static_cache = isinstance(past_key_values, StaticCache) - - # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward - if ( - self.config._attn_implementation == "sdpa" - and not using_static_cache - and not output_attentions - and head_mask is None - and alibi is None - ): - if AttentionMaskConverter._ignore_causal_mask_sdpa( - attention_mask, - inputs_embeds=input_tensor, - past_key_values_length=past_seen_tokens, - is_training=self.training, - ): - return None - - dtype, device = input_tensor.dtype, input_tensor.device - # difference from original, replace torch.finfo(dtype).min to float16 for prevent overflow for fp16/bf16 execution - min_dtype = torch.finfo(torch.float16).min - batch_size, sequence_length, _ = input_tensor.shape - if using_static_cache: - target_length = past_key_values.get_max_length() - else: - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else past_seen_tokens + sequence_length - ) - - # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). - causal_mask = _falcon_prepare_4d_causal_attention_mask_with_cache_position( - attention_mask, - sequence_length=sequence_length, - target_length=target_length, - dtype=dtype, - device=device, - min_dtype=min_dtype, - cache_position=cache_position, - batch_size=input_tensor.shape[0], - ) - - # We take care to integrate alibi bias in the causal_mask here - if head_mask is None and alibi is not None: - alibi = alibi.reshape(batch_size, -1, *alibi.shape[1:]) - causal_mask = torch.masked_fill( - alibi / math.sqrt(self.config.hidden_size // self.num_heads), - causal_mask < -1, - min_dtype, - ) - - if ( - self.config._attn_implementation == "sdpa" - and attention_mask is not None - and attention_mask.device.type == "cuda" - and not output_attentions - ): - # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when - # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. - # Details: https://github.com/pytorch/pytorch/issues/110213 - causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) - - return causal_mask - - class FalconModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() patch_cos_sin_cached_fp32(self._model.transformer) - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask"): - self._model.transformer._update_causal_mask_original = self._model.transformer._update_causal_mask - self._model.transformer._update_causal_mask = types.MethodType( - _falcon_update_causal_mask, self._model.transformer - ) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - - if is_transformers_version("<", "4.53") and hasattr(self._model.transformer, "_update_causal_mask_original"): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - - -class GptNeoxModelPatcher(OVDecoderModelPatcher): - def __enter__(self): - super().__enter__() - - if ( - is_transformers_version("<", "4.53") - and hasattr(self._model, "transformer") - and hasattr(self._model.transformer, "_update_causal_mask") - ): - self._model.transformer._update_causal_mask_original = self._model.transformer._update_causal_mask - self._model.transformer._update_causal_mask = types.MethodType( - _falcon_update_causal_mask, self._model.transformer - ) - def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if ( - is_transformers_version("<", "4.53") - and hasattr(self._model, "transformer") - and hasattr(self._model.transformer, "_update_causal_mask_original") - ): - self._model.transformer._update_causal_mask = self._model.transformer._update_causal_mask_original - del self._model.transformer._update_causal_mask_original - # Adopted from https://github.com/huggingface/optimum/blob/v1.24.0/optimum/bettertransformer/models/attention.py#L96 def _gptj_attn(self, query, key, value, attention_mask=None, head_mask=None): @@ -3084,13 +2480,13 @@ def _bloom_attn_forward( alibi: torch.Tensor, attention_mask: torch.Tensor, layer_past=None, - head_mask: Optional[torch.Tensor] = None, use_cache: bool = False, output_attentions: bool = False, - cache_position: Optional[torch.LongTensor] = None, + **kwargs, ): from transformers.models.bloom.modeling_bloom import dropout_add + head_mask = kwargs.get("head_mask", None) if head_mask is not None or output_attentions: return self._orig_forward( hidden_states, @@ -3098,10 +2494,9 @@ def _bloom_attn_forward( alibi, attention_mask, layer_past=layer_past, - head_mask=head_mask, use_cache=use_cache, output_attentions=output_attentions, - cache_position=cache_position, + **kwargs, ) batch_size, q_length, _ = hidden_states.shape # [batch_size, seq_length, 3 x hidden_size] @@ -3110,13 +2505,13 @@ def _bloom_attn_forward( query_layer, key_layer, value_layer = self._reshape(fused_qkv) if layer_past is not None: - cache_kwargs = {"cache_position": cache_position} - key_layer, value_layer = layer_past.update(key_layer, value_layer, self.layer_idx, cache_kwargs) + cache_kwargs = {k: kwargs[k] for k in ("cache_position",) if k in kwargs} + key_layer, value_layer = layer_past.update(key_layer, value_layer, self.layer_idx, cache_kwargs or None) alibi = alibi.reshape(batch_size, -1, *alibi.shape[1:]) if attention_mask is not None: # no matter the length, we just slice it - kv_length = cache_position[-1] + 1 # cache position is 0-indexed while length should start from 1 + kv_length = key_layer.shape[-2] causal_mask = attention_mask[:, :, :, :kv_length] alibi = torch.masked_fill(alibi, causal_mask.bool(), torch.finfo(alibi.dtype).min) @@ -3526,22 +2921,6 @@ def __enter__(self): ): self._model.config._orig_attn_implementation = self._model.config._attn_implementation self._model.config._attn_implementation = "sdpa" - if self._model.config.model_type == "qwen2" and is_transformers_version("<", "4.48"): - from transformers.models.qwen2.modeling_qwen2 import QWEN2_ATTENTION_CLASSES - - sdpa_attn = QWEN2_ATTENTION_CLASSES["sdpa"] - - for layer in self._model.model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) - - if self._model.config.model_type == "llama" and is_transformers_version("<", "4.47"): - from transformers.models.llama.modeling_llama import LLAMA_ATTENTION_CLASSES - - sdpa_attn = LLAMA_ATTENTION_CLASSES["sdpa"] - for layer in self._model.model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) if self._internal_patcher is not None: return self._internal_patcher.__enter__() @@ -4507,115 +3886,69 @@ def __exit__(self, exc_type, exc_value, traceback): def patch_qwen2vl_vision_blocks(model, force_new_behaviour=False): - if not force_new_behaviour and is_transformers_version("<=", "4.48.99"): - # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L390 - # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) - def sdpa_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - rotary_pos_emb: torch.Tensor = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> torch.Tensor: - from transformers.models.qwen2_vl.modeling_qwen2_vl import apply_rotary_pos_emb_vision - - seq_length = hidden_states.shape[0] - q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - - if is_transformers_version(">=", "4.49"): - if position_embeddings is None: - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - cos = emb.cos().float() - sin = emb.sin().float() - else: - cos, sin = position_embeddings - q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) - else: - q = apply_rotary_pos_emb_vision(q.unsqueeze(0), rotary_pos_emb).squeeze(0) - k = apply_rotary_pos_emb_vision(k.unsqueeze(0), rotary_pos_emb).squeeze(0) - - q = q.transpose(0, 1) - k = k.transpose(0, 1) - v = v.transpose(0, 1) - attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) - attn_output = attn_output.transpose(0, 1) - attn_output = attn_output.reshape(seq_length, -1) - attn_output = self.proj(attn_output) - return attn_output - - # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L430 - # added attention_mask input propagation to self.attn - def block_forward(self, hidden_states, attention_mask, rotary_pos_emb) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), attention_mask=attention_mask, rotary_pos_emb=rotary_pos_emb - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states - - else: - # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L391 - # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) - def sdpa_attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: torch.Tensor, - rotary_pos_emb: torch.Tensor = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ): - def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - def apply_rotary_pos_emb_vision( - q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor - ) -> Tuple[torch.Tensor, torch.Tensor]: - orig_q_dtype = q.dtype - orig_k_dtype = k.dtype - q, k = q.float(), k.float() - cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - q_embed = q_embed.to(orig_q_dtype) - k_embed = k_embed.to(orig_k_dtype) - return q_embed, k_embed - - seq_length = hidden_states.shape[0] - q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) - if position_embeddings is None: - emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) - cos = emb.cos().float() - sin = emb.sin().float() - else: - cos, sin = position_embeddings - q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) - q = q.transpose(0, 1) - k = k.transpose(0, 1) - v = v.transpose(0, 1) - attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) - attn_output = attn_output.transpose(0, 1) - attn_output = attn_output.reshape(seq_length, -1) - attn_output = self.proj(attn_output) - return attn_output - - # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L446 - # added attention_mask input propagation to self.attn - def block_forward( - self, - hidden_states, - attention_mask, - rotary_pos_emb: Optional[torch.Tensor] = None, - position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - ) -> torch.Tensor: - hidden_states = hidden_states + self.attn( - self.norm1(hidden_states), - attention_mask=attention_mask, - rotary_pos_emb=rotary_pos_emb, - position_embeddings=position_embeddings, - ) - hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) - return hidden_states + # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L391 + # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len) + def sdpa_attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + rotary_pos_emb: torch.Tensor = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ): + def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + def apply_rotary_pos_emb_vision( + q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + return q_embed, k_embed + + seq_length = hidden_states.shape[0] + q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0) + if position_embeddings is None: + emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1) + cos = emb.cos().float() + sin = emb.sin().float() + else: + cos, sin = position_embeddings + q, k = apply_rotary_pos_emb_vision(q, k, cos, sin) + q = q.transpose(0, 1) + k = k.transpose(0, 1) + v = v.transpose(0, 1) + attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0) + attn_output = attn_output.transpose(0, 1) + attn_output = attn_output.reshape(seq_length, -1) + attn_output = self.proj(attn_output) + return attn_output + + # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L446 + # added attention_mask input propagation to self.attn + def block_forward( + self, + hidden_states, + attention_mask, + rotary_pos_emb: Optional[torch.Tensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states), + attention_mask=attention_mask, + rotary_pos_emb=rotary_pos_emb, + position_embeddings=position_embeddings, + ) + hidden_states = hidden_states + self.mlp(self.norm2(hidden_states)) + return hidden_states for block in model.blocks: block._orig_forward = block.forward @@ -4904,58 +4237,41 @@ def patched_forward(*args, **kwargs): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.53.0"): - # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min - # to avoid overflow issues on some hardware (e.g. Intel NPU) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) + # for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min + # to avoid overflow issues on some hardware (e.g. Intel NPU) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) - # for decoder models, we use eager mask without vmap for sdpa as well - # to avoid a nan output issue in OpenVINO that only happens in case of: - # non-stateful models on cpu and stateful models on npu - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # for decoder models, we use eager mask without vmap for sdpa as well + # to avoid a nan output issue in OpenVINO that only happens in case of: + # non-stateful models on cpu and stateful models on npu + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) class SanaTextEncoderModelPatcher(ModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version("<", "4.47.0"): - from transformers.models.gemma2.modeling_gemma2 import GEMMA2_ATTENTION_CLASSES - - sdpa_attn = GEMMA2_ATTENTION_CLASSES["sdpa"] - for layer in self._model.layers: - layer.self_attn._orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(sdpa_attn.forward, layer.self_attn) - else: - self._model.config._orig_attn_implementation = self._model.config._attn_implementation - self._model.config._attn_implementation = "sdpa" + self._model.config._orig_attn_implementation = self._model.config._attn_implementation + self._model.config._attn_implementation = "sdpa" - if is_transformers_version(">=", "4.53"): - # starting from 4.53, we get unmatching outputs if we use the boolean mask - # TODO: This is an openvino issue (inconsistency between boolean and float masks) - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) + # starting from 4.53, we get unmatching outputs if we use the boolean mask + # TODO: This is an openvino issue (inconsistency between boolean and float masks) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.47.0"): - for layer in self._model.layers: - layer.self_attn.forward = layer.self_attn._orig_forward - del layer.self_attn._orig_forward - else: - self._model.config._attn_implementation = self._model.config._orig_attn_implementation - del self._model.config._orig_attn_implementation + self._model.config._attn_implementation = self._model.config._orig_attn_implementation + del self._model.config._orig_attn_implementation - if is_transformers_version(">=", "4.53"): - # remove the eager_mask_without_vmap from the ALL_MASK_ATTENTION_FUNCTIONS - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + # remove the eager_mask_without_vmap from the ALL_MASK_ATTENTION_FUNCTIONS + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) class MiniCPMModelPatcher(OVDecoderModelPatcher): @@ -5007,56 +4323,6 @@ def patched_forward(*args, **kwargs): self.patched_forward = patched_forward -# Adopted from https://github.com/huggingface/transformers/blob/v4.49.0-Gemma-3/src/transformers/models/gemma3/modeling_gemma3.py#L1147 -def _gemma3_mm_update_causal_mask( - self, attention_mask, token_type_ids, past_key_values, cache_position, input_tensor, is_training: bool = False -): - if attention_mask is not None and attention_mask.dim() == 4: - # In this case we assume that the mask comes already in inverted - # form and requires no inversion or slicing. - return attention_mask - - min_dtype = torch.finfo(torch.float16).min - inputs_lead_dim, sequence_length = input_tensor.shape[:2] - target_length = ( - attention_mask.shape[-1] - if isinstance(attention_mask, torch.Tensor) - else cache_position[0] + sequence_length + 1 - ) - - causal_mask = torch.full( - (sequence_length, target_length), fill_value=min_dtype, dtype=self.dtype, device=cache_position.device - ) - - # Causal diagonal mask only if training, otherwise attend to the whole prefix. Training-specific attn for prefix is handled below - if sequence_length != 1: - causal_mask = torch.triu(causal_mask, diagonal=1) - - causal_mask *= torch.arange(target_length, device=cache_position.device) > cache_position.reshape(-1, 1) - causal_mask = causal_mask[None, None, :, :].expand(inputs_lead_dim, 1, -1, -1) - - # Apply bidirectional mask on images if token type ids are provided - if token_type_ids is not None and sequence_length != 1: - token_type_mask = token_type_ids.unsqueeze(1) == token_type_ids.unsqueeze(2) - token_type_mask[token_type_ids == 0] = False # if text token do not change anything - token_type_mask = token_type_mask.unsqueeze(1).to(causal_mask.device, dtype=torch.bool) - causal_mask = causal_mask.clone() - causal_mask[:, :, :, :sequence_length] = causal_mask[:, :, :, :sequence_length].masked_fill( - token_type_mask, 0.0 - ) - - if attention_mask is not None: - causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit - mask_length = attention_mask.shape[-1] - - # Then apply padding mask (will mask pad tokens) - padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :].to(causal_mask.device) - padding_mask = padding_mask == 0 - causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill(padding_mask, min_dtype) - - return causal_mask - - class Gemma3LMModelPatcher(OVDecoderModelPatcher): def __init__( self, @@ -5064,79 +4330,28 @@ def __init__( model: "PreTrainedModel", model_kwargs: Optional[Dict[str, Any]] = None, ): - # Difference from original: - # uses Dynamic cache from legacy cache instead of HybridCache - # calculate causal mask from multimodal - - def forward( - self, attention_mask, position_ids, past_key_values, token_type_ids, inputs_embeds, use_cache=True - ): - if is_transformers_version("<", "5"): - pkv = DynamicCache.from_legacy_cache(past_key_values) - else: - pkv = DynamicCache(past_key_values) - - past_seen_tokens = past_key_values[0][0].shape[-2] - cache_position = torch.arange( - past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device - ) - forward_kwargs = {} - - if is_transformers_version("<", "4.52"): - attention_mask = self._update_causal_mask_mm( - attention_mask, token_type_ids, past_key_values, cache_position, inputs_embeds - ) - else: - forward_kwargs["token_type_ids"] = token_type_ids - - result = self.__orig_forward( - input_ids=None, - attention_mask=attention_mask, - position_ids=position_ids, - cache_position=cache_position, - past_key_values=pkv, - inputs_embeds=inputs_embeds, - use_cache=use_cache, - **forward_kwargs, - ) - upd_pkv = result["past_key_values"] - result["past_key_values"] = postprocess_past_key_values(upd_pkv) - return result - - if is_transformers_version("<", "4.53.0"): - model.__orig_forward = model.forward - model.forward = types.MethodType(forward, model) - - super().__init__(config, model, model_kwargs) - - def __enter__(self): - super().__enter__() - - if is_transformers_version("<", "4.52.0"): - self._model._update_causal_mask_mm = types.MethodType(_gemma3_mm_update_causal_mask, self._model) - elif ( - is_transformers_version("<", "4.53.0") - and hasattr(self._model, "model") - and hasattr(self._model.model, "_update_causal_mask") - ): - self._model.model._orig_update_causual_mask = self._model.model._update_causal_mask - self._model.model._update_causal_mask = types.MethodType(_gemma3_mm_update_causal_mask, self._model.model) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) + super().__init__(config, model, model_kwargs) - if is_transformers_version("<", "4.53.0"): - self._model.forward = self._model.__orig_forward + model_forward = self.orig_forward + + # precompute the token_type_ids bidirectional (image) mask since transformers v5.6 + # (https://github.com/huggingface/transformers/pull/45454) is_first_iteration removed + # in create_causal_mask_mapping + @functools.wraps(model_forward) + def forward_with_precomputed_mask(*args, **kwargs): + bound_args = inspect.signature(model_forward).bind(*args, **kwargs) + bound_args.apply_defaults() + inputs_embeds = bound_args.arguments.get("inputs_embeds") + token_type_ids = bound_args.arguments.get("token_type_ids") + attention_mask = bound_args.arguments.get("attention_mask") + if token_type_ids is not None and isinstance(attention_mask, torch.Tensor): + sliding_window = self._model.config.get_text_config().sliding_window + bound_args.arguments["attention_mask"] = _create_gemma4_unified_bidirectional_mask_dict( + attention_mask, token_type_ids, inputs_embeds, sliding_window + ) + return model_forward(*bound_args.args, **bound_args.kwargs) - if is_transformers_version("<", "4.52"): - del self._update_causal_mask_mm - elif ( - is_transformers_version("<", "4.53.0") - and hasattr(self._model, "model") - and hasattr(self._model.model, "_orig_update_causual_mask") - ): - self._model.model._update_causal_mask = self._model.model._orig_update_causual_mask - del self._model.model._orig_update_causual_mask + self.orig_forward = forward_with_precomputed_mask # Forward method of the language model of Gemma3n, needs to be patched to pass 'per_layer_inputs', @@ -5220,11 +4435,12 @@ def gemma3n_language_model_forward( return outputs -# Creates a dict of causal masks with bidirectional attention for vision tokens -# on sliding_attention layers, matching the behavior of transformers -# create_causal_mask_mapping when use_bidirectional_attention == "vision". +# Creates a dict of causal masks with bidirectional attention for vision tokens, +# matching the behavior of transformers create_masks_for_generate with +# block_sequence_ids when use_bidirectional_attention == "vision" (the bidirectional +# overlay applies to both the full-attention and sliding-attention masks). # Needs to be patched to pass proper 'sliding_mask' for prefill stage. -# Original code: https://github.com/huggingface/transformers/blob/v5.5.0/src/transformers/models/gemma4/modeling_gemma4.py#L1986 +# Original code: https://github.com/huggingface/transformers/blob/v5.10.0/src/transformers/models/gemma4/modeling_gemma4.py#L2320 def _create_gemma4_bidirectional_mask_dict(attention_mask_2d, mm_token_type_ids, inputs_embeds, sliding_window): dtype = inputs_embeds.dtype device = inputs_embeds.device @@ -5272,7 +4488,9 @@ def _create_gemma4_bidirectional_mask_dict(attention_mask_2d, mm_token_type_ids, same_group = (query_groups.unsqueeze(2) == key_groups.unsqueeze(1)) & (key_groups.unsqueeze(1) >= 0) same_group = same_group.unsqueeze(1) # [batch, 1, seq_len, total_len] - # Undo masking for same-group vision tokens in sliding mask + # Un-mask same-group vision tokens in both masks (bidirectional attention within an image). + if is_transformers_version(">=", "5.9"): + full_mask = full_mask.masked_fill(same_group, 0.0) sliding_mask = sliding_mask.masked_fill(same_group, 0.0) return { @@ -5480,6 +4698,10 @@ def gemma4_text_attention_forward( ) -> tuple: from transformers.models.gemma4.modeling_gemma4 import apply_rotary_pos_emb as apply_rotary_pos_emb_gemma4 + # since transformers >= v5.6 (PR #45788) `shared_kv_states` dict and is passed and `kv_shared_layer_index` removed + shared_kv_states = kwargs.pop("shared_kv_states", None) + legacy_shared_kv_states = hasattr(self, "kv_shared_layer_index") + input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.head_dim) @@ -5490,8 +4712,12 @@ def gemma4_text_attention_forward( query_states = apply_rotary_pos_emb_gemma4(query_states, cos, sin, unsqueeze_dim=2) query_states = query_states.transpose(1, 2) - if self.is_kv_shared_layer and past_key_values is not None: - key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + if self.is_kv_shared_layer and (not legacy_shared_kv_states or past_key_values is not None): + if legacy_shared_kv_states: + key_states, value_states = past_key_values.shared_layers[self.kv_shared_layer_index] + else: + key_states, value_states = shared_kv_states[self.layer_type] + key_states = key_states.to(query_states.device) value_states = value_states.to(query_states.device) else: @@ -5506,18 +4732,26 @@ def gemma4_text_attention_forward( value_states = value_states.transpose(1, 2) if past_key_values is not None: - cache_kwargs = { - "sin": sin, - "cos": cos, - "cache_position": cache_position, - "sliding_window": self.sliding_window, - } - if not self.is_kv_shared_layer: - key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx, cache_kwargs) - if self.store_full_length_kv: - if not hasattr(past_key_values, "shared_layers"): - past_key_values.shared_layers = {} - past_key_values.shared_layers[self.layer_idx] = key_states, value_states + if legacy_shared_kv_states: + cache_kwargs = { + "sin": sin, + "cos": cos, + "cache_position": cache_position, + "sliding_window": self.sliding_window, + } + if not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + if self.store_full_length_kv: + if not hasattr(past_key_values, "shared_layers"): + past_key_values.shared_layers = {} + past_key_values.shared_layers[self.layer_idx] = key_states, value_states + else: + if not self.is_kv_shared_layer: + key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx) + if self.store_full_length_kv and shared_kv_states is not None: + shared_kv_states[self.layer_type] = key_states, value_states attention_interface = gemma4_eager_attention_forward_patched @@ -5746,14 +4980,10 @@ def embeddings_forward( nb_patches_h = p_attn_mask[:, 0].sum() nb_patches_w = p_attn_mask[0].sum() - if is_transformers_version("<", "4.55"): - fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h) - fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w) - else: - h_indices = torch.arange(nb_patches_h, device=pixel_values.device, dtype=pixel_values.dtype) - w_indices = torch.arange(nb_patches_w, device=pixel_values.device, dtype=pixel_values.dtype) - fractional_coords_h = h_indices / nb_patches_h * (1 - 1e-6) - fractional_coords_w = w_indices / nb_patches_w * (1 - 1e-6) + h_indices = torch.arange(nb_patches_h, device=pixel_values.device, dtype=pixel_values.dtype) + w_indices = torch.arange(nb_patches_w, device=pixel_values.device, dtype=pixel_values.dtype) + fractional_coords_h = h_indices / nb_patches_h * (1 - 1e-6) + fractional_coords_w = w_indices / nb_patches_w * (1 - 1e-6) bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True) bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True) @@ -5839,202 +5069,6 @@ def __exit__(self, exc_type, exc_value, traceback): layer.self_attn.forward = layer.self_attn._orig_forward -# Adopted from https://github.com/huggingface/optimum/blob/main/optimum/bettertransformer/models/decoder_models.py#L367 -def _blenderbot_attn_forward_legacy( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - if output_attentions or layer_head_mask is not None: - return self._orig_forward( - hidden_states, key_value_states, past_key_value, attention_mask, layer_head_mask, output_attentions - ) - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) - # get key, value proj - # `past_key_value[0].shape[2] == key_value_states.shape[1]` - # is checking that the `sequence_length` of the `past_key_value` is the same as - # the provided `key_value_states` to support prefix tuning - if is_cross_attention and past_key_value is not None and past_key_value[0].shape[2] == key_value_states.shape[1]: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - query_states = self._shape(query_states, tgt_len, bsz) - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.dropout if self.training else 0.0, - is_causal=False, - ) - - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, None, past_key_value - - -# Adopted from https://github.com/huggingface/transformers/blob/v4.52.3/src/transformers/models/blenderbot/modeling_blenderbot.py#L156 -def _blenderbot_attn_forward_new( - self, - hidden_states: torch.Tensor, - key_value_states=None, - past_key_value=None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, - cache_position: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - if output_attentions or layer_head_mask is not None: - return self._orig_forward( - hidden_states, - key_value_states, - past_key_value, - attention_mask, - layer_head_mask, - output_attentions, - cache_position, - ) - is_cross_attention = key_value_states is not None - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states).view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - query_states = query_states - - if past_key_value is not None: - if isinstance(past_key_value, EncoderDecoderCache): - is_updated = past_key_value.is_updated.get(self.layer_idx) - if is_cross_attention: - # after the first generated id, we can subsequently re-use all key/value_states from cache - curr_past_key_value = past_key_value.cross_attention_cache - else: - curr_past_key_value = past_key_value.self_attention_cache - else: - curr_past_key_value = past_key_value - - current_states = key_value_states if is_cross_attention else hidden_states - if is_cross_attention and past_key_value is not None and is_updated: - # reuse k,v, cross_attentions - key_states = curr_past_key_value.key_cache[self.layer_idx] - value_states = curr_past_key_value.value_cache[self.layer_idx] - else: - key_states = self.k_proj(current_states) - value_states = self.v_proj(current_states) - key_states = key_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - value_states = value_states.view(bsz, -1, self.num_heads, self.head_dim).transpose(1, 2) - - if past_key_value is not None: - # save all key/value_states to cache to be re-used for fast auto-regressive generation - cache_position = cache_position if not is_cross_attention else None - key_states, value_states = curr_past_key_value.update( - key_states, value_states, self.layer_idx, {"cache_position": cache_position} - ) - # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls - if is_cross_attention: - past_key_value.is_updated[self.layer_idx] = True - - proj_shape = (bsz, self.num_heads, -1, self.head_dim) - # difference with original, removed query_states = query_states.reshape(*proj_shape) * self.scale as scale is part of SDPA - query_states = query_states.reshape(*proj_shape) - key_states = key_states.reshape(*proj_shape) - value_states = value_states.reshape(*proj_shape) - - # Difference with original, use SDPA instead of eager attention - - attn_output = torch.nn.functional.scaled_dot_product_attention( - query_states, - key_states, - value_states, - attn_mask=attention_mask, - dropout_p=self.dropout if self.training else 0.0, - is_causal=False, - ) - - if attn_output.size() != (bsz, self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - outputs = (attn_output, None) - - if is_transformers_version("<", "4.54"): - outputs += (past_key_value,) - - return outputs - - -if is_transformers_version(">=", "4.52"): - _blenderbot_attn_forward = _blenderbot_attn_forward_new -else: - _blenderbot_attn_forward = _blenderbot_attn_forward_legacy - - def modulewise_patch(model, module_cls, patch_forward): for _, module in model.named_children(): if isinstance(module, module_cls): @@ -6056,54 +5090,6 @@ def modulewise_unpatch(model, module_cls): modulewise_unpatch(module, module_cls) -class BlenderbotModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot.modeling_blenderbot import BlenderbotAttention - - modulewise_patch(self._model, BlenderbotAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot.modeling_blenderbot import BlenderbotAttention - - modulewise_unpatch(self._model, BlenderbotAttention) - - -class BlenderbotSmallModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot_small.modeling_blenderbot_small import BlenderbotSmallAttention - - modulewise_patch(self._model, BlenderbotSmallAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.blenderbot_small.modeling_blenderbot_small import BlenderbotSmallAttention - - modulewise_unpatch(self._model, BlenderbotSmallAttention) - - -class PegasusModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version("<", "4.56"): - from transformers.models.pegasus.modeling_pegasus import PegasusAttention - - modulewise_patch(self._model, PegasusAttention, _blenderbot_attn_forward) - - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version("<", "4.56"): - from transformers.models.pegasus.modeling_pegasus import PegasusAttention - - modulewise_unpatch(self._model, PegasusAttention) - - # Copied from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py#L596 # In 4.52.0, the loop is only over hitted experts, but we need to loop over all experts for tracing def _qwen2moe_sparse_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -6155,44 +5141,26 @@ class Qwen2MoEPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.52.0"): - if is_transformers_version("<", "5"): - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock + if is_transformers_version("<", "5"): + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - modulewise_patch(self._model, Qwen2MoeSparseMoeBlock, _qwen2moe_sparse_block_forward) - else: - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts + modulewise_patch(self._model, Qwen2MoeSparseMoeBlock, _qwen2moe_sparse_block_forward) + else: + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - self.original_moe_forward = Qwen2MoeExperts.forward - Qwen2MoeExperts.forward = lfm2_moe_experts_forward + self.original_moe_forward = Qwen2MoeExperts.forward + Qwen2MoeExperts.forward = lfm2_moe_experts_forward def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.52.0"): - if is_transformers_version("<", "5"): - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - - modulewise_unpatch(self._model, Qwen2MoeSparseMoeBlock) - else: - from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - - Qwen2MoeExperts.forward = self.original_moe_forward - - -class MarianModelPatcher(OVSeq2SeqModelPatcher): - def __enter__(self): - super().__enter__() - if is_transformers_version(">=", "4.49.0") and is_transformers_version("<", "4.56"): - from transformers.models.marian.modeling_marian import MarianAttention - - modulewise_patch(self._model, MarianAttention, _blenderbot_attn_forward) + if is_transformers_version("<", "5"): + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock - def __exit__(self, exc_type, exc_value, traceback): - super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.49.0") and is_transformers_version("<", "4.56"): - from transformers.models.marian.modeling_marian import MarianAttention + modulewise_unpatch(self._model, Qwen2MoeSparseMoeBlock) + else: + from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeExperts - modulewise_unpatch(self._model, MarianAttention) + Qwen2MoeExperts.forward = self.original_moe_forward # Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L698 @@ -6222,210 +5190,6 @@ def speecht5_decoder_prenet_forward( return inputs_embeds -# Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L889 -# this is a patch to avoid CPU plugin issue that is happened on 16-th iteration of token generation -# values computed by self-attention attn_output = torch.bmm(attn_probs, value_states) in a decoder gets incorrect -def speecht5_attention_forward( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - position_bias: Optional[torch.Tensor] = None, - output_attentions: bool = False, - serialize: bool = False, -) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - is_cross_attention = key_value_states is not None - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) * self.scaling - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_states = value_states.view(*proj_shape) - - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" - f" {attn_weights.size()}" - ) - - # relative attention bias - if position_bias is not None: - reshape_q = query_states.contiguous().view(bsz * self.num_heads, -1, self.head_dim).transpose(0, 1) - rel_pos_bias = torch.matmul(reshape_q, position_bias.transpose(-2, -1)) - rel_pos_bias = rel_pos_bias.transpose(0, 1).view( - bsz * self.num_heads, position_bias.size(0), position_bias.size(1) - ) - attn_weights += rel_pos_bias - - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1) - - if layer_head_mask is not None: - if layer_head_mask.size() != (self.num_heads,): - raise ValueError( - f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" - ) - attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - if output_attentions: - # this operation is a bit awkward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to be reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None - - attn_probs = torch.nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) - - # this is a patch to avoid CPU plugin issue!!! - # issue is happened on 16-th iteration of token generation - # since 16-th iteration of token generation, values computed by self-attention in a decoder gets incorrect - eps = 1e-30 - attn_output = torch.bmm(attn_probs + eps, value_states) - - if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned across GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights_reshaped, past_key_value - - -# Adopted from https://github.com/huggingface/transformers/blob/v4.51.3/src/transformers/models/speecht5/modeling_speecht5.py#L1121 -# this is a patch for a model to avoid incorrect tracing -# cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple are computed using encoder_hidden_states -def speecht5_decoder_layer_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - encoder_hidden_states: Optional[torch.Tensor] = None, - encoder_attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - cross_attn_layer_head_mask: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - output_attentions: Optional[bool] = False, - use_cache: Optional[bool] = True, - serialize: bool = False, -): - residual = hidden_states - - # Self Attention - # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 - self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None - # add present self-attn cache to positions 1,2 of present_key_value tuple - hidden_states, self_attn_weights, present_key_value = self.self_attn( - hidden_states=hidden_states, - past_key_value=self_attn_past_key_value, - attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - output_attentions=output_attentions, - serialize=serialize, - ) - - hidden_states = self.dropout(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - # Cross-Attention Block - cross_attn_present_key_value = None - cross_attn_weights = None - if encoder_hidden_states is not None: - residual = hidden_states - - # this is a patch for a model to avoid incorrect tracing!!! - # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple - # are computed using encoder_hidden_states - if past_key_value is not None and len(past_key_value) > 3: - cross_attn_past_key_value = past_key_value[-2:] - else: - cross_attn_past_key_value = None - hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn( - hidden_states=hidden_states, - key_value_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - layer_head_mask=cross_attn_layer_head_mask, - past_key_value=cross_attn_past_key_value, - output_attentions=output_attentions, - ) - hidden_states = self.dropout(hidden_states) - hidden_states = residual + hidden_states - hidden_states = self.encoder_attn_layer_norm(hidden_states) - - # add cross-attn to positions 3,4 of present_key_value tuple - present_key_value = present_key_value + cross_attn_present_key_value - - # Fully Connected - hidden_states = hidden_states + self.feed_forward(hidden_states) - hidden_states = self.final_layer_norm(hidden_states) - - outputs = (hidden_states,) - - if output_attentions: - outputs += (self_attn_weights, cross_attn_weights) - - if use_cache: - outputs += (present_key_value,) - - return outputs - - class SpeechT5ModelPatcher(ModelPatcher): def __enter__(self): if self.real_config._behavior != "vocoder": @@ -6435,12 +5199,6 @@ def __enter__(self): self._model.speecht5.decoder.prenet.forward = types.MethodType( speecht5_decoder_prenet_forward, self._model.speecht5.decoder.prenet ) - if is_transformers_version("<", "4.54"): - for layer in self._model.speecht5.decoder.wrapped_decoder.layers: - layer.__orig_forward = layer.forward - layer.forward = types.MethodType(speecht5_decoder_layer_forward, layer) - layer.self_attn.__orig_forward = layer.self_attn.forward - layer.self_attn.forward = types.MethodType(speecht5_attention_forward, layer.self_attn) def __exit__(self, exc_type, exc_value, traceback): if self.real_config._behavior != "vocoder": @@ -6449,10 +5207,6 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.speecht5.decoder.prenet.forward = types.MethodType( self._model.speecht5.decoder.prenet.__orig_forward, self._model.speecht5.decoder.prenet ) - if is_transformers_version("<", "4.54"): - for layer in self._model.speecht5.decoder.wrapped_decoder.layers: - layer.forward = types.MethodType(layer.__orig_forward, layer) - layer.self_attn.forward = types.MethodType(layer.self_attn.__orig_forward, layer.self_attn) def __init__( self, @@ -7131,44 +5885,6 @@ def llama4_attn_forward( return attn_output, attn_weights -# modified from https://github.com/huggingface/transformers/blob/v4.51.0/src/transformers/models/llama4/modeling_llama4.py#L157 -# due to openvino transformations issue removed routed_out.view(-1, hidden_dim) in scatter_add_ -def llama4_moe_forward(self, hidden_states): - batch, seq_len, hidden_dim = hidden_states.shape - hidden_states = hidden_states.view(-1, self.hidden_dim) - router_logits = self.router(hidden_states).transpose(0, 1) - tokens_per_expert = batch * seq_len - - router_top_value, router_indices = torch.topk(router_logits.transpose(0, 1), self.top_k, dim=1) - router_scores = ( - torch.full_like(router_logits.transpose(0, 1), float("-inf")) - .scatter_(1, router_indices, router_top_value) - .transpose(0, 1) - ) - # We do this to make sure we have -inf for non topK tokens before going through the ! - # Here we are just creating a tensor to index each and every single one of the hidden states. Let s maybe register a buffer for this! - router_indices = ( - torch.arange(tokens_per_expert, device=hidden_states.device).view(1, -1).expand(router_scores.size(0), -1) - ) - router_scores = torch.sigmoid(router_scores.float()).to(hidden_states.dtype) - - router_indices = router_indices.reshape(-1, 1).expand(-1, hidden_dim) - routed_in = torch.gather( - input=hidden_states, - dim=0, - index=router_indices, - ).to(hidden_states.device) - # we gather inputs corresponding to each expert based on the router indices - routed_in = routed_in * router_scores.transpose(0, 1).reshape(-1, 1) - routed_out = self.experts(routed_in) - out = self.shared_expert(hidden_states) - # now that we finished expert computation -> we scatter add because we gathered previously - # we have to do this because we used all experts on all tokens. This is faster than the for loop, tho you are compute bound - # this scales a lot better if you do EP! - out.scatter_add_(dim=0, index=router_indices, src=routed_out) - return out, router_scores - - # Copied from https://github.com/huggingface/transformers/blob/v4.56.0/src/transformers/masking_utils.py#L105 # transformers.masking_utils._legacy_chunked_overlay deprecated since transformers v5 def _legacy_chunked_overlay(chunk_size: int) -> Callable: @@ -7185,9 +5901,6 @@ def __enter__(self): self._model.model.rotary_emb._orig_forward = self._model.model.rotary_emb.forward self._model.model.rotary_emb.forward = types.MethodType(llama4_rope_forward, self._model.model.rotary_emb) for layer in self._model.model.layers[: self._model.model.config.num_hidden_layers]: - if layer.is_moe_layer and is_transformers_version("<", "4.54"): - layer.feed_forward._orig_forward = layer.feed_forward.forward - layer.feed_forward.forward = types.MethodType(llama4_moe_forward, layer.feed_forward) layer.self_attn._orig_forward = layer.self_attn.forward layer.self_attn.forward = types.MethodType(llama4_attn_forward, layer.self_attn) @@ -7203,8 +5916,6 @@ def __exit__(self, exc_type, exc_value, traceback): self._model.model.rotary_emb.forward = self._model.model.rotary_emb._orig_forward for layer in self._model.model.layers[: self._model.model.config.num_hidden_layers]: - if layer.is_moe_layer and is_transformers_version("<", "4.54"): - layer.feed_forward.forward = layer.feed_forward._orig_forward layer.self_attn.forward = layer.self_attn._orig_forward if is_transformers_version(">=", "4.56"): @@ -7540,26 +6251,24 @@ class Qwen3MoeModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.53"): - if is_transformers_version("<", "5"): - self.original_moe_forward = Qwen3MoeSparseMoeBlock.forward - Qwen3MoeSparseMoeBlock.forward = qwen3_moe_forward_patched - else: - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + if is_transformers_version("<", "5"): + self.original_moe_forward = Qwen3MoeSparseMoeBlock.forward + Qwen3MoeSparseMoeBlock.forward = qwen3_moe_forward_patched + else: + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts - self.original_moe_forward = Qwen3MoeExperts.forward - Qwen3MoeExperts.forward = lfm2_moe_experts_forward + self.original_moe_forward = Qwen3MoeExperts.forward + Qwen3MoeExperts.forward = lfm2_moe_experts_forward def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.53"): - if is_transformers_version("<", "5"): - Qwen3MoeSparseMoeBlock.forward = self.original_moe_forward - else: - from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + if is_transformers_version("<", "5"): + Qwen3MoeSparseMoeBlock.forward = self.original_moe_forward + else: + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts - Qwen3MoeExperts.forward = self.original_moe_forward + Qwen3MoeExperts.forward = self.original_moe_forward # The original implementation of this forward method can be found at: @@ -8008,8 +6717,8 @@ def ov_causal_conv1d(conv_state, input_embeds, weight, bias): state_len = conv_state.shape[-1] groups = hidden_size // w_in_channels - input_embeds_new = torch.cat([conv_state, input_embeds], dim=-1).to(weight.dtype) - conv_out = F.conv1d(input_embeds_new, weight, bias, padding=0, groups=groups) + input_embeds_new = torch.cat([conv_state, input_embeds.to(conv_state.dtype)], dim=-1) + conv_out = F.conv1d(input_embeds_new.to(weight.dtype), weight, bias, padding=0, groups=groups) conv_out = conv_out[:, :, -seq_len:] new_conv_state = input_embeds_new[:, :, -state_len:] @@ -8040,6 +6749,9 @@ def lfm2_short_conv_forward_patched( # in transformers < v5 attention_mask was never applied in Lfm2ShortConv https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/lfm2/modeling_lfm2.py#L485 # until a fix was added in https://github.com/huggingface/transformers/pull/41790/ if is_transformers_version(">=", "5"): + # since transformers v5.4, Lfm2ShortConv.slow_forward passes attention_mask as 3 positional arg + if attention_mask is None and is_transformers_version(">=", "5.4"): + attention_mask = cache_position dtype = x.dtype is_decoding = torch.tensor(seqlen == 1, dtype=dtype) x = (x * (attention_mask[:, :seqlen, None] * (1 - is_decoding) + is_decoding)).to(dtype) @@ -8115,14 +6827,21 @@ def update( """ # Update the cache layer_idx = self.attention_layer_idx_mapping[layer_idx] + compute_dtype = key_states.dtype + if self.key_cache[layer_idx].numel() == 0: self.key_cache[layer_idx] = key_states self.value_cache[layer_idx] = value_states else: - self.key_cache[layer_idx] = torch.cat([self.key_cache[layer_idx], key_states], dim=-2) - self.value_cache[layer_idx] = torch.cat([self.value_cache[layer_idx], value_states], dim=-2) + cache_dtype = self.key_cache[layer_idx].dtype + self.key_cache[layer_idx] = torch.cat( + [self.key_cache[layer_idx], key_states.to(cache_dtype)], dim=-2 + ) + self.value_cache[layer_idx] = torch.cat( + [self.value_cache[layer_idx], value_states.to(cache_dtype)], dim=-2 + ) - return self.key_cache[layer_idx], self.value_cache[layer_idx] + return self.key_cache[layer_idx].to(compute_dtype), self.value_cache[layer_idx].to(compute_dtype) def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: """ @@ -8254,23 +6973,21 @@ class GptOssModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - if is_transformers_version(">=", "4.55.0"): - if is_transformers_version("<", "5"): - from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts + if is_transformers_version("<", "5"): + from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts - self.original_gpt_oss_forward = GptOssExperts.forward - GptOssExperts.forward = gpt_oss_forward - else: - register_ov_batched_mm(self) + self.original_gpt_oss_forward = GptOssExperts.forward + GptOssExperts.forward = gpt_oss_forward + else: + register_ov_batched_mm(self) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) - if is_transformers_version(">=", "4.55.0"): - if is_transformers_version("<", "5"): - from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts + if is_transformers_version("<", "5"): + from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts - GptOssExperts.forward = self.original_gpt_oss_forward + GptOssExperts.forward = self.original_gpt_oss_forward # This patch overrides the following line in Transformers: @@ -8524,7 +7241,13 @@ def __exit__(self, exc_type, exc_value, traceback): def afmoe_moe_forward_patched(self, hidden_states): num_experts = self.config.num_experts batch_size, seq_len, hidden_dim = hidden_states.shape - routing_weights, selected_experts = self.router(hidden_states, self.expert_bias) + # transformers >= v5.4 (PR #44063): router returns (router_logits, top_scores, selected_experts) + # transformers < v5.4: router returns (routing_weights, selected_experts) + router_output = self.router(hidden_states, self.expert_bias) + if len(router_output) == 3: + _, routing_weights, selected_experts = router_output + else: + routing_weights, selected_experts = router_output new_routing_weights = torch.zeros(batch_size * seq_len, self.config.num_experts, dtype=routing_weights.dtype) new_routing_weights.scatter_(dim=1, index=selected_experts, src=routing_weights) hidden_states = hidden_states.view(-1, hidden_dim) @@ -8537,7 +7260,7 @@ def afmoe_moe_forward_patched(self, hidden_states): hidden_states = hidden_states.repeat(num_experts, 1) hidden_states = hidden_states.view(num_experts, -1, hidden_dim) - act_fn = self.experts[0].act_fn + act_fn = self.experts.act_fn if is_transformers_version(">=", "5.4") else self.experts[0].act_fn # compute experts outputs in a vectorized form gate = torch.bmm(hidden_states, self.gate_projs.transpose(1, 2)) @@ -8569,17 +7292,26 @@ def __enter__(self): # with bf16 weights that leads to operands types mismatch in torch.bmm during TorchScript tracing # Now we align with hidden_states (that will be always fp32 due to patching # above for embedding layer during tracing) - afmoe_moe.down_projs = torch.concat( - tuple(afmoe_moe.experts[i].down_proj.weight.unsqueeze(0) for i in range(num_experts)), - dim=0, - ) - afmoe_moe.gate_projs = torch.concat( - tuple(afmoe_moe.experts[i].gate_proj.weight.unsqueeze(0) for i in range(num_experts)), - dim=0, - ) - afmoe_moe.up_projs = torch.concat( - tuple(afmoe_moe.experts[i].up_proj.weight.unsqueeze(0) for i in range(num_experts)), dim=0 - ) + + # transformers >= v5.4 (PR #44063): AfmoeExperts stores stacked gate_up_proj/down_proj + # transformers < v5.4: AfmoeExperts is a ModuleList of AfmoeMLP with separate gate/up/down projs + if hasattr(afmoe_moe.experts, "gate_up_proj"): + intermediate_dim = afmoe_moe.experts.gate_up_proj.shape[1] // 2 + afmoe_moe.gate_projs = afmoe_moe.experts.gate_up_proj[:, :intermediate_dim, :].detach() + afmoe_moe.up_projs = afmoe_moe.experts.gate_up_proj[:, intermediate_dim:, :].detach() + afmoe_moe.down_projs = afmoe_moe.experts.down_proj.detach() + else: + afmoe_moe.down_projs = torch.concat( + tuple(afmoe_moe.experts[i].down_proj.weight.unsqueeze(0) for i in range(num_experts)), + dim=0, + ).detach() + afmoe_moe.gate_projs = torch.concat( + tuple(afmoe_moe.experts[i].gate_proj.weight.unsqueeze(0) for i in range(num_experts)), + dim=0, + ).detach() + afmoe_moe.up_projs = torch.concat( + tuple(afmoe_moe.experts[i].up_proj.weight.unsqueeze(0) for i in range(num_experts)), dim=0 + ).detach() if is_openvino_version("<", "2026.1.0"): afmoe_moe.down_projs = afmoe_moe.down_projs.float() @@ -8798,14 +7530,23 @@ def forward( if position_ids is None: position_ids = cache_position.unsqueeze(0) - causal_mask = create_causal_mask( - config=self.config, - input_embeds=inputs_embeds, - attention_mask=attention_mask, - cache_position=cache_position, - past_key_values=past_key_values, - position_ids=position_ids, - ) + if is_transformers_version(">=", "5.5"): + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + past_key_values=past_key_values, + position_ids=position_ids, + ) + else: + causal_mask = create_causal_mask( + config=self.config, + input_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) if hidden_states is None: hidden_states = torch.zeros( @@ -9458,8 +8199,9 @@ def _create_gemma4_unified_bidirectional_mask_dict( same_group = (query_groups.unsqueeze(2) == key_groups.unsqueeze(1)) & (key_groups.unsqueeze(1) >= 0) same_group = same_group.unsqueeze(1) # [batch, 1, seq_len, total_len] - # Un-mask same-group vision tokens in both masks (bidirectional attention within an image). - full_mask = full_mask.masked_fill(same_group, 0.0) + # Un-mask same-group vision tokens in both masks (bidirectional attention within an image) + if is_transformers_version(">=", "5.9"): + full_mask = full_mask.masked_fill(same_group, 0.0) sliding_mask = sliding_mask.masked_fill(same_group, 0.0) return { diff --git a/optimum/exporters/openvino/patching_utils.py b/optimum/exporters/openvino/patching_utils.py index d8ce0a2c05..a50047c974 100644 --- a/optimum/exporters/openvino/patching_utils.py +++ b/optimum/exporters/openvino/patching_utils.py @@ -23,43 +23,24 @@ import transformers from transformers import PreTrainedModel from transformers.cache_utils import DynamicCache, EncoderDecoderCache +from transformers.masking_utils import ( + ALL_MASK_ATTENTION_FUNCTIONS, + _ignore_causal_mask_sdpa, + and_masks, + causal_mask_function, + eager_mask, + find_packed_sequence_indices, + padding_mask_function, + prepare_padding_mask, + sdpa_mask, +) from transformers.modeling_outputs import BaseModelOutput from optimum.exporters.base import ExporterConfig +from optimum.exporters.openvino._traceable_decorator import traceable_check_model_inputs from optimum.intel.utils.import_utils import is_transformers_version -if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - from optimum.exporters.openvino._traceable_cache import TraceableCache - - -if is_transformers_version(">=", "4.48"): - from transformers.cache_utils import DynamicCache, EncoderDecoderCache -if is_transformers_version(">=", "4.53"): - from transformers.masking_utils import ( - ALL_MASK_ATTENTION_FUNCTIONS, - _ignore_causal_mask_sdpa, - and_masks, - causal_mask_function, - eager_mask, - padding_mask_function, - prepare_padding_mask, - sdpa_mask, - ) - - -if is_transformers_version(">=", "4.53.1"): - from transformers.masking_utils import find_packed_sequence_indices - - -if is_transformers_version(">=", "4.54"): - from transformers.utils import TransformersKwargs - - from optimum.exporters.openvino._traceable_decorator import traceable_check_model_inputs -else: - TransformersKwargs = object - - if is_transformers_version(">=", "4.56"): import transformers.masking_utils from transformers.cache_utils import DynamicLayer @@ -86,18 +67,14 @@ def override_arguments(args, kwargs, forward_signature, model_kwargs: dict[str, def preprocess_encoder_outputs(encoder_outputs): - if is_transformers_version(">=", "4.54") and isinstance(encoder_outputs, (list, tuple)): + if isinstance(encoder_outputs, (list, tuple)): encoder_outputs = BaseModelOutput(*encoder_outputs) return encoder_outputs def preprocess_past_key_values(past_key_values): - if ( - is_transformers_version(">=", "4.48") - and isinstance(past_key_values, (list, tuple)) - and isinstance(past_key_values[0], (list, tuple)) - ): + if isinstance(past_key_values, (list, tuple)) and isinstance(past_key_values[0], (list, tuple)): if len(past_key_values[0]) == 2: if hasattr(DynamicCache, "from_legacy_cache"): past_key_values = DynamicCache.from_legacy_cache(past_key_values) @@ -162,12 +139,6 @@ def find_packed_sequence_indices_patched(position_ids: torch.Tensor) -> torch.Te return torch.zeros_like(position_ids) -if is_transformers_version(">=", "4.53"): - _prepare_padding_mask_slice = "_slice" in inspect.signature(prepare_padding_mask).parameters -else: - _prepare_padding_mask_slice = False - - # Custom vectorized implementation of sdpa_mask without using vmap def _orig_sdpa_mask_without_vmap( batch_size: int, @@ -185,7 +156,7 @@ def _orig_sdpa_mask_without_vmap( q_length = cache_position.shape[0] # Potentially pad the 2D mask, and slice it correctly - if _prepare_padding_mask_slice: + if "_slice" in inspect.signature(prepare_padding_mask).parameters: padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset, _slice=False) else: padding_mask = prepare_padding_mask(attention_mask, kv_length, kv_offset) @@ -215,38 +186,38 @@ def _orig_sdpa_mask_without_vmap( # Compatibility wrapper for sdpa_mask_without_vmap from optimum. # The installed optimum version expects (batch_size, cache_position: Tensor, kv_length, ...), # but transformers >= 5.5 passes (batch_size, q_length: int, kv_length: int, q_offset: int, ...). -def sdpa_mask_without_vmap(batch_size, q_length=None, kv_length=None, q_offset=0, kv_offset=0, **kwargs): - import inspect - - sig = inspect.signature(_orig_sdpa_mask_without_vmap) - if is_transformers_version(">=", "5.5") and "cache_position" in sig.parameters and q_length is not None: - # Old optimum signature: (batch_size, cache_position, kv_length, kv_offset, ...) - cache_position = torch.arange(q_length, dtype=torch.long) + q_offset - kwargs.pop("q_offset", None) - kwargs.pop("allow_is_bidirectional_skip", None) - kwargs.pop("allow_torch_fix", None) - kwargs.pop("use_vmap", None) - kwargs.pop("device", None) - return _orig_sdpa_mask_without_vmap(batch_size, cache_position, kv_length, kv_offset=kv_offset, **kwargs) +def sdpa_mask_without_vmap(**kwargs): + kwargs.pop("use_vmap", None) + if is_transformers_version("<", "5"): + return _orig_sdpa_mask_without_vmap(**kwargs) + elif ( + is_transformers_version(">=", "5.4") + and is_transformers_version("<", "5.9") + and isinstance(kwargs.get("q_length", None), torch.Tensor) + ): + q_length = kwargs.pop("q_length", None) + q_offset = kwargs.pop("q_offset", 0) + cache_position = torch.arange(q_offset, q_offset + q_length, device=q_length.device) + return sdpa_mask(q_length=cache_position, use_vmap=False, **kwargs) else: - return _orig_sdpa_mask_without_vmap( - batch_size, q_length=q_length, kv_length=kv_length, q_offset=q_offset, kv_offset=kv_offset, **kwargs - ) + return sdpa_mask(use_vmap=False, **kwargs) # Adapted from https://github.com/huggingface/transformers/blob/v4.53.0/src/transformers/masking_utils.py#L433 # Specifically for OpenVINO, we use torch.finfo(torch.float16).min instead of torch.finfo(dtype).min -def eager_mask_without_vmap(*args, **kwargs): +def eager_mask_without_vmap(**kwargs): kwargs.pop("allow_is_causal_skip", None) - dtype = kwargs.get("dtype", torch.float32) - mask = sdpa_mask_without_vmap(*args, allow_is_causal_skip=False, **kwargs) - # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not - # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf - mask = torch.where( - mask, - torch.tensor(0.0, device=mask.device, dtype=dtype), - torch.tensor(torch.finfo(torch.float16).min, device=mask.device, dtype=dtype), - ) + kwargs.pop("allow_torch_fix", None) + dtype = kwargs.pop("dtype", torch.float32) + mask = sdpa_mask_without_vmap(allow_is_causal_skip=False, allow_torch_fix=False, **kwargs) + if mask is not None: + # we use torch.finfo(torch.float16).min instead torch.finfo(dtype).min to avoid an overflow but not + # sure this is the right way to handle this, we are basically pretending that -65,504 is -inf + mask = torch.where( + mask, + torch.tensor(0.0, device=mask.device, dtype=dtype), + torch.tensor(torch.finfo(torch.float16).min, device=mask.device, dtype=dtype), + ) return mask @@ -293,7 +264,7 @@ def __init__( self.orig_forward_name = "forward" if hasattr(self._model, "forward") else "call" self.orig_forward = getattr(self._model, self.orig_forward_name) - if is_transformers_version(">=", "4.54") and hasattr(self.orig_forward, "__wrapped__"): + if hasattr(self.orig_forward, "__wrapped__"): # the original check_model_inputs has some failing cases that we fix in traceable_check_model_inputs # we fix those issues in a PR in transformers https://github.com/huggingface/transformers/pull/40811 # issues are: support for positional args (use_cache for instance) and fix for _CAN_RECORD_REGISTRY @@ -348,7 +319,7 @@ def patched_forward(*args, **kwargs): # format of outputs. It is common for the output type of a model to vary, such as tensor, list, # tuple, etc. For Transformers models, the output is encapsulated in a ModelOutput object that # contains the output names of the model. In the case of Timm classification models, the output - # is of type tensor. By default, it is assumed that the output names mentioned in the ONNX config + # is of type tensor. By default, it is assumed that the output names mentioned in the OpenVINO config # match the outputs in order. filtered_outputs = {} output_names = list(config.outputs.keys()) @@ -370,7 +341,7 @@ def patched_forward(*args, **kwargs): raise ValueError( f"{config.__class__.__name__} expects the model to return {num_outputs} outputs: {output_names_str}, " f"but the it returned a single output of type {type(outputs)}. Please make sure either that the model " - "returns all the expected outputs, or that the ONNX config is correctly defined with the expected outputs." + "returns all the expected outputs, or that the OpenVINO config is correctly defined with the expected outputs." ) output_name = output_names[0] filtered_outputs[output_name] = outputs @@ -396,25 +367,16 @@ def __enter__(self): self.patch_ops() setattr(self._model, self.orig_forward_name, self.patched_forward) - # This is a workaround for the Cache class in transformers, we replace it - # with traceable cache is because the original one used in transformers - # inherited from nn.Module (for a couple versions), which can't be traced as input. - if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - self.original_cache_class = transformers.cache_utils.Cache - transformers.cache_utils.Cache = TraceableCache - # This is a workaround for mask generation in transformers >= 4.53. # The masking process uses vmap which is not traceable by TorchScript. - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask_without_vmap) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask_without_vmap) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask_without_vmap) # This is a workaround for the find_packed_sequence_indices function in transformers which # should only return a tensor of zeros with the same shape as position_ids indicating no packed sequence indices. # The function uses torch.diff which is not traceable by TorchScript. - if is_transformers_version(">=", "4.53.1"): - self.original_find_packed_sequence_indices = find_packed_sequence_indices - transformers.masking_utils.find_packed_sequence_indices = find_packed_sequence_indices_patched + self.original_find_packed_sequence_indices = find_packed_sequence_indices + transformers.masking_utils.find_packed_sequence_indices = find_packed_sequence_indices_patched # Starting from transformers 4.56.0, DynamicCache uses DynamicLayer which has an update method # that uses torch.cat to concatenate an empty tensor with the key/value states during the first call. @@ -427,15 +389,10 @@ def __exit__(self, exc_type, exc_value, traceback): self.restore_ops() setattr(self._model, self.orig_forward_name, self.orig_forward) - if is_transformers_version(">=", "4.44") and is_transformers_version("<", "4.50"): - transformers.cache_utils.Cache = self.original_cache_class - - if is_transformers_version(">=", "4.53"): - ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) - ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) + ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) - if is_transformers_version(">=", "4.53.1"): - transformers.masking_utils.find_packed_sequence_indices = self.original_find_packed_sequence_indices + transformers.masking_utils.find_packed_sequence_indices = self.original_find_packed_sequence_indices if is_transformers_version(">=", "4.56"): DynamicLayer.update = self.original_dynamic_layer_update diff --git a/optimum/intel/__init__.py b/optimum/intel/__init__.py index 23ef819efe..90289ede94 100644 --- a/optimum/intel/__init__.py +++ b/optimum/intel/__init__.py @@ -44,6 +44,7 @@ "OVModelForQuestionAnswering", "OVModelForSeq2SeqLM", "OVModelForSpeechSeq2Seq", + "OVModelForImageTextToText", "OVModelForTextToSpeechSeq2Seq", "OVModelForVision2Seq", "OVModelForVisualCausalLM", @@ -167,6 +168,7 @@ OVModelForCustomTasks, OVModelForFeatureExtraction, OVModelForImageClassification, + OVModelForImageTextToText, OVModelForMaskedLM, OVModelForQuestionAnswering, OVModelForSeq2SeqLM, diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index f6b02fcb3f..dbaaddaccb 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -87,7 +87,13 @@ OVModelOpenCLIPVisual, ) from .modeling_sam import OVSamModel -from .modeling_seq2seq import OVModelForPix2Struct, OVModelForSeq2SeqLM, OVModelForSpeechSeq2Seq, OVModelForVision2Seq +from .modeling_seq2seq import ( + OVModelForImageTextToText, + OVModelForPix2Struct, + OVModelForSeq2SeqLM, + OVModelForSpeechSeq2Seq, + OVModelForVision2Seq, +) from .modeling_text2speech import OVModelForTextToSpeechSeq2Seq from .modeling_visual_language import OVModelForVisualCausalLM diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index eadbbf130f..aa75996152 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -679,7 +679,8 @@ def _update_model_kwargs_for_generation( outputs=outputs, model_kwargs=model_kwargs, is_encoder_decoder=is_encoder_decoder, **kwargs ) - if "position_ids" in model_kwargs: + # _prepare_position_ids_for_generation will infer position ids since transformers v5.2 + if "position_ids" in model_kwargs and not hasattr(self, "_prepare_position_ids_for_generation"): position_ids = model_kwargs["position_ids"] new_position_id = position_ids[..., -1:].clone() new_position_id += 1 @@ -1472,12 +1473,11 @@ def _update_model_kwargs_for_generation( self, outputs: ModelOutput, model_kwargs: Dict[str, Any], num_new_tokens: int = 1, **kwargs ) -> Dict[str, Any]: model_kwargs["cache_params"] = outputs.get("cache_params", None) - if ( - model_kwargs.get("use_cache", True) - and "cache_position" in model_kwargs - and model_kwargs["cache_position"] is not None - ): - model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens + if model_kwargs.get("use_cache", True): + if "cache_position" in model_kwargs and model_kwargs["cache_position"] is not None: + model_kwargs["cache_position"] = model_kwargs["cache_position"][-1:] + num_new_tokens + elif model_kwargs.get("cache_params") is not None: + model_kwargs["cache_position"] = torch.tensor([num_new_tokens], dtype=torch.long) if "attention_mask" in model_kwargs: attention_mask = model_kwargs["attention_mask"] @@ -1500,12 +1500,16 @@ def prepare_inputs_for_generation( # Overwitten -- uses `cache_params` as opposed to `past_key_values` if self.use_cache: - # `cache_position` should have been initialized in `generate` if cache_position is None: - raise ValueError( - "`cache_position` should not be None as it should have been initialized in " - "`model.generate`, you are responsible for passing in a valid `cache_position` if " - "you are calling `prepare_inputs_for_generation` directly with `use_cache=True`" + if is_transformers_version("<", "5.4"): + raise ValueError( + "`cache_position` should not be None as it should have been initialized in " + "`model.generate`, you are responsible for passing in a valid `cache_position` if " + "you are calling `prepare_inputs_for_generation` directly with `use_cache=True`" + ) + # infer from cache_params: None means prefill (0), otherwise means decoding stage + cache_position = torch.tensor( + [0 if cache_params is None else 1], device=input_ids.device, dtype=torch.long ) if cache_position[0] > 0: # decoding stage so it takes the last token diff --git a/optimum/intel/openvino/modeling_seq2seq.py b/optimum/intel/openvino/modeling_seq2seq.py index fc084f11f7..3169ff05f4 100644 --- a/optimum/intel/openvino/modeling_seq2seq.py +++ b/optimum/intel/openvino/modeling_seq2seq.py @@ -13,6 +13,7 @@ # limitations under the License. import logging import os +import warnings from pathlib import Path from typing import Dict, List, Optional, Tuple, Union @@ -23,8 +24,11 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from openvino import CompiledModel, Core from openvino._offline_transformations import apply_moc_transformations, compress_model_transformation +from torch import nn +from torch.nn import functional as F from transformers import ( AutoConfig, + AutoModelForImageTextToText, AutoModelForSeq2SeqLM, AutoModelForSpeechSeq2Seq, GenerationConfig, @@ -55,21 +59,6 @@ ) -# AutoModelForVision2Seq is deprecated since v4.54 -# https://github.com/huggingface/transformers/blob/v4.54.0/src/transformers/models/auto/modeling_auto.py#L2151 -if is_transformers_version(">=", "4.54.0"): - from transformers import AutoModelForImageTextToText - - transformers_auto_class = AutoModelForImageTextToText -else: - from transformers import AutoModelForVision2Seq - - transformers_auto_class = AutoModelForVision2Seq - -from torch import nn -from torch.nn import functional as F - - core = Core() logger = logging.getLogger(__name__) @@ -910,6 +899,8 @@ def forward( # Add the attention_mask inputs when needed if "attention_mask" in self.input_names: + if attention_mask is None: + attention_mask = torch.ones_like(inputs[self.main_input_name]) inputs["attention_mask"] = attention_mask # Qwen3-ASR requires input_features chunking before passing to encoder for processing of long audios. @@ -1104,8 +1095,8 @@ def _reorder_cache( """, INPUTS_DOCSTRING, ) -class OVModelForVision2Seq(OVModelForSeq2SeqLM): - auto_model_class = transformers_auto_class +class OVModelForImageTextToText(OVModelForSeq2SeqLM): + auto_model_class = AutoModelForImageTextToText main_input_name = "pixel_values" export_feature = "image-to-text" @@ -1163,7 +1154,7 @@ def prepare_inputs_for_generation( + IMAGE_TO_TEXT_EXAMPLE.format( processor_class=_PROCESSOR_FOR_DOC, tokenizer_class=_TOKENIZER_FOR_DOC, - model_class="OVModelForVision2Seq", + model_class="OVModelForImageTextToText", checkpoint="microsoft/trocr-small-handwritten", ) ) @@ -1209,15 +1200,15 @@ def _reshape(self, model: openvino.Model, batch_size: int, sequence_length: int, """, INPUTS_DOCSTRING, ) -class OVModelForPix2Struct(OVModelForVision2Seq): +class OVModelForPix2Struct(OVModelForImageTextToText): auto_model_class = Pix2StructForConditionalGeneration main_input_name = "flattened_patches" export_feature = "image-to-text" - # this is needed to avoid circular calls when OVModelForVision2Seq is called to instantiate a OVModelForPix2Struct + # this is needed to avoid circular calls when OVModelForImageTextToText is called to instantiate a OVModelForPix2Struct @classmethod def _from_pretrained(cls, model_id: Union[str, Path], config: "PretrainedConfig", **kwargs): - return super(OVModelForVision2Seq, cls)._from_pretrained(model_id, config, **kwargs) + return super(OVModelForImageTextToText, cls)._from_pretrained(model_id, config, **kwargs) def prepare_inputs_for_generation( self, @@ -1293,6 +1284,18 @@ def _reshape(self, model: openvino.Model, batch_size: int, sequence_length: int, return model +class OVModelForVision2Seq(OVModelForImageTextToText): + @classmethod + def from_pretrained(cls, *args, **kwargs): + warnings.warn( + "OVModelForVision2Seq is deprecated and will be removed in a future release. " + "Use OVModelForImageTextToText instead.", + FutureWarning, + stacklevel=2, + ) + return super().from_pretrained(*args, **kwargs) + + @add_start_docstrings( """ Speech Sequence-to-sequence model with a language modeling head for OpenVINO inference. This class officially supports whisper, speech_to_text. @@ -1571,18 +1574,3 @@ def prepare_inputs_for_generation( "decoder_position_ids": decoder_position_ids, "cache_position": cache_position, } - - def _get_logits_processor(self, generation_config: GenerationConfig, *args, **kwargs): - # Whisper uses forced_decoder_ids for default task and language specification, while original _get_logits_processor does not allow it - # see for details https://github.com/huggingface/transformers/issues/37172 - if not hasattr(generation_config, "forced_decoder_ids") or is_transformers_version(">=", "4.53.0"): - # since transformers 4.53.0, forced_decoder_ids is deprecated: https://github.com/huggingface/transformers/pull/38232 - logits_processor = super()._get_logits_processor(generation_config, *args, **kwargs) - else: - forced_decoder_ids = generation_config.forced_decoder_ids - - if is_transformers_version(">=", "4.50.0"): - generation_config.forced_decoder_ids = None - logits_processor = super()._get_logits_processor(generation_config, *args, **kwargs) - generation_config.forced_decoder_ids = forced_decoder_ids - return logits_processor diff --git a/optimum/intel/openvino/modeling_visual_language.py b/optimum/intel/openvino/modeling_visual_language.py index 440acb0ae5..6575d30314 100644 --- a/optimum/intel/openvino/modeling_visual_language.py +++ b/optimum/intel/openvino/modeling_visual_language.py @@ -30,33 +30,40 @@ PreTrainedTokenizer, ) from transformers.modeling_outputs import BaseModelOutputWithPooling -from transformers.models.qwen2_vl.modeling_qwen2_vl import VisionRotaryEmbedding +from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLModel +from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLModel, VisionRotaryEmbedding +from transformers.models.qwen3_vl.modeling_qwen3_vl import ( + Qwen3VLModel, + Qwen3VLVisionModel, + Qwen3VLVisionRotaryEmbedding, +) from transformers.utils import ModelOutput -from ...exporters.openvino import main_export -from ...exporters.openvino.stateful import ensure_stateful_is_available, model_has_input_output_name -from ...exporters.openvino.utils import save_config -from ..utils.import_utils import is_transformers_version -from .configuration import OVConfig, OVQuantizationConfigBase, OVWeightQuantizationConfig -from .modeling_base import OVBaseModel, OVModelPart -from .modeling_decoder import CausalLMOutputWithPast, OVModelForCausalLM -from .utils import ( +from optimum.exporters.openvino import main_export +from optimum.exporters.openvino.stateful import ensure_stateful_is_available, model_has_input_output_name +from optimum.exporters.openvino.utils import save_config +from optimum.intel.openvino.configuration import OVConfig, OVQuantizationConfigBase, OVWeightQuantizationConfig +from optimum.intel.openvino.modeling_base import OVBaseModel, OVModelPart +from optimum.intel.openvino.modeling_decoder import CausalLMOutputWithPast, OVModelForCausalLM +from optimum.intel.openvino.utils import ( OV_LANGUAGE_MODEL_NAME, OV_TEXT_EMBEDDINGS_MODEL_NAME, OV_VISION_EMBEDDINGS_MODEL_NAME, TemporaryDirectory, classproperty, ) +from optimum.intel.utils.import_utils import is_transformers_version -if is_transformers_version(">=", "4.46.0"): - from transformers import AutoModelForImageTextToText +if is_transformers_version(">=", "5.2"): + from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5Model, + Qwen3_5VisionModel, + Qwen3_5VisionRotaryEmbedding, + ) - transformers_auto_class = AutoModelForImageTextToText -else: - from transformers import AutoModelForVision2Seq - transformers_auto_class = AutoModelForVision2Seq +from transformers import AutoModelForImageTextToText if TYPE_CHECKING: @@ -196,7 +203,7 @@ def prepare_inputs( if self.config.model_type in ["qwen3_5", "qwen3_5_moe"] and position_ids.ndim != 3: position_ids = np.repeat(np.expand_dims(position_ids, 0), 4, axis=0) - elif self.config.model_type in ["qwen2_vl", "qwen3_vl"] and position_ids.ndim != 3: + elif (self.config.model_type in ["qwen2_vl", "qwen2_5_vl", "qwen3_vl"]) and position_ids.ndim != 3: position_ids = np.repeat(np.expand_dims(position_ids, 0), 3, axis=0) inputs["position_ids"] = position_ids @@ -378,7 +385,7 @@ def forward(self, audio_feature, audio_mask): class OVModelForVisualCausalLM(OVBaseModel, GenerationMixin): export_feature = "image-text-to-text" additional_parts = [] - auto_model_class = transformers_auto_class + auto_model_class = AutoModelForImageTextToText @classproperty def _all_ov_model_paths(cls) -> Dict[str, str]: @@ -1161,7 +1168,7 @@ def preprocess_inputs( else: prompt = text - if is_transformers_version(">", "4.47.99") and getattr(processor, "patch_size", None) is None: + if getattr(processor, "patch_size", None) is None: if ( getattr(config, "vision_config", None) is not None and getattr(config.vision_config, "patch_size", None) is not None @@ -1584,7 +1591,7 @@ def preprocess_inputs( if video is not None and "