diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index f3d8fed9de..d2bfda8db2 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -409,7 +409,7 @@ def main_export( # for avoiding confusion we disable remote code for them if ( trust_remote_code - and model_type in {"falcon", "mpt", "phi"} + and model_type in {"falcon", "mpt", "phi", "phi3"} and ("with-past" in task or original_task == "auto") and not custom_export_configs ): diff --git a/optimum/exporters/openvino/model_configs.py b/optimum/exporters/openvino/model_configs.py index 32ee9ed288..00bda2b486 100644 --- a/optimum/exporters/openvino/model_configs.py +++ b/optimum/exporters/openvino/model_configs.py @@ -912,7 +912,7 @@ class Phi3OpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig): MistralDummyPastKeyValuesGenerator, ) + TextDecoderOpenVINOConfig.DUMMY_INPUT_GENERATOR_CLASSES DUMMY_PKV_GENERATOR_CLASS = MistralDummyPastKeyValuesGenerator - MIN_TRANSFORMERS_VERSION = "4.36.0" + MIN_TRANSFORMERS_VERSION = "4.49.0" _MODEL_PATCHER = Phi3ModelPatcher diff --git a/optimum/exporters/openvino/model_patcher.py b/optimum/exporters/openvino/model_patcher.py index 199bbce903..2ad7124231 100644 --- a/optimum/exporters/openvino/model_patcher.py +++ b/optimum/exporters/openvino/model_patcher.py @@ -458,12 +458,24 @@ def patch_cos_sin_cached_fp32(model): # Adapted from https://github.com/huggingface/transformers/blob/3c307e380ad07ca16903a39e09a47d532cb782d9/src/transformers/models/phimoe/modular_phimoe.py#L57 -def _longrope_forward(self, x, position_ids=None, layer_type=None): +def _longrope_forward(self, x, position_ids=None, layer_type=None, **kwargs): # _compute_longrope_parameters https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/modeling_rope_utils.py#L391 - self.config.standardize_rope_params() - rope_parameters = ( - self.config.rope_parameters[layer_type] if layer_type is not None else self.config.rope_parameters - ) + # transformers >= 5 stores RoPE settings under config.rope_parameters; transformers < 5 (e.g. 4.57) stores + # them under config.rope_scaling and as plain attributes on the config. + if hasattr(self.config, "rope_parameters") and self.config.rope_parameters is not None: + rope_parameters = ( + self.config.rope_parameters[layer_type] if layer_type is not None else self.config.rope_parameters + ) + else: + rope_scaling = getattr(self.config, "rope_scaling", None) or {} + rope_parameters = dict(rope_scaling) + rope_parameters.setdefault("rope_theta", getattr(self.config, "rope_theta", 10000.0)) + rope_parameters.setdefault( + "original_max_position_embeddings", + getattr(self.config, "original_max_position_embeddings", self.config.max_position_embeddings), + ) + rope_parameters.setdefault("partial_rotary_factor", getattr(self.config, "partial_rotary_factor", 1.0)) + rope_theta = rope_parameters["rope_theta"] long_factor = rope_parameters["long_factor"] short_factor = rope_parameters["short_factor"] @@ -474,9 +486,11 @@ def _longrope_forward(self, x, position_ids=None, layer_type=None): head_dim = getattr(self.config, "head_dim", self.config.hidden_size // self.config.num_attention_heads) dim = int(head_dim * partial_rotary_factor) - seq_len = torch.max(position_ids) + 1 + # needed for transformers < v5 for phimoe + seq_len = kwargs.get("seq_len", None) + _seq_len = seq_len if position_ids is None else torch.max(position_ids) + 1 # bool tensor to avoid only one path getting traced - is_long = seq_len > original_max + is_long = _seq_len > original_max # Compute the inverse frequencies -- scaled based on the target sequence length long_factors = torch.tensor(long_factor, dtype=torch.float32, device=x.device) @@ -498,12 +512,20 @@ def _longrope_forward(self, x, position_ids=None, layer_type=None): attention_factor = 1.0 if factor <= 1.0 else math.sqrt(1 + math.log(factor) / math.log(original_max)) mscale = attention_factor - # https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/phimoe/modeling_phimoe.py#L116 - inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) - position_ids_expanded = position_ids[:, None, :].float() - device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" - with torch.autocast(device_type=device_type, enabled=False): - freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + if is_transformers_version(">=", "5") or position_ids is not None: + # https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/phimoe/modeling_phimoe.py#L116 + inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device) + position_ids_expanded = position_ids[:, None, :].float() + device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * mscale + sin = emb.sin() * mscale + else: + # needed for transformers < v5 for phimoe + t = torch.arange(_seq_len, device=x.device, dtype=torch.float32) + freqs = torch.outer(t, inv_freq) emb = torch.cat((freqs, freqs), dim=-1) cos = emb.cos() * mscale sin = emb.sin() * mscale @@ -532,12 +554,14 @@ def __enter__(self): # non-stateful models on cpu and stateful models on npu ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap) - if is_transformers_version(">=", "5"): - for module in self._model.modules(): - rope_type = getattr(module, "rope_type", None) - if rope_type == "longrope" and isinstance(getattr(module, "config", None), RotaryEmbeddingConfigMixin): - module._rope_orig_forward = module.forward - module.forward = types.MethodType(_longrope_forward, module) + for module in self._model.modules(): + rope_type = getattr(module, "rope_type", None) + if rope_type == "longrope" and ( + is_transformers_version("<", "5") + or isinstance(getattr(module, "config", None), RotaryEmbeddingConfigMixin) + ): + module._rope_orig_forward = module.forward + module.forward = types.MethodType(_longrope_forward, module) def __exit__(self, exc_type, exc_value, traceback): super().__exit__(exc_type, exc_value, traceback) @@ -550,11 +574,10 @@ def __exit__(self, exc_type, exc_value, traceback): ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask) ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask) - if is_transformers_version(">=", "5"): - for module in self._model.modules(): - if hasattr(module, "_rope_orig_forward"): - module.forward = module._rope_orig_forward - del module._rope_orig_forward + for module in self._model.modules(): + if hasattr(module, "_rope_orig_forward"): + module.forward = module._rope_orig_forward + del module._rope_orig_forward def _mixtral_sparse_moe_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -1828,11 +1851,8 @@ class Phi3ModelPatcher(OVDecoderModelPatcher): def __enter__(self): super().__enter__() - # currently, long RoPE can not be traced for long context support, disable it for avoid potential accuracy issues - if self._model.config.max_position_embeddings != getattr( - self._model.config, "original_max_position_embeddings", self._model.config.max_position_embeddings - ): - self._model.config.max_position_embeddings = self._model.config.original_max_position_embeddings + # LongRoPE is handled via _longrope_forward in OVDecoderModelPatcher for both transformers >= 5 and < 5, + # so keep the original max_position_embeddings to preserve the correct attention scaling factor. if is_transformers_version("<", "4.48.0"): self._model.model._orig_forward = self._model.model.forward diff --git a/optimum/intel/openvino/modeling_decoder.py b/optimum/intel/openvino/modeling_decoder.py index 69f7b83ee0..6f0bd6ce70 100644 --- a/optimum/intel/openvino/modeling_decoder.py +++ b/optimum/intel/openvino/modeling_decoder.py @@ -898,6 +898,8 @@ def _from_pretrained( init_cls = OVBloomForCausalLM elif model_type == "gpt_bigcode": init_cls = OVGPTBigCodeForCausalLM + elif model_type == "phi3": + init_cls = OVPhi3ForCausalLM elif model_type in SSM_MODELS: init_cls = OVModelWithMambaForCausalLM else: @@ -950,6 +952,48 @@ def _from_pretrained( return causal_model +class OVPhi3ForCausalLM(OVModelForCausalLM): + # Adapted from https://github.com/huggingface/transformers/blob/v4.57.0/src/transformers/models/phi3/modeling_phi3.py#L493 + def prepare_inputs_for_generation( + self, + input_ids, + past_key_values=None, + attention_mask=None, + inputs_embeds=None, + cache_position=None, + position_ids=None, + use_cache=True, + logits_to_keep=None, + **kwargs, + ): + # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the + # process + + # When the first time input length reached long and short factor switching point, enforce re-compute cache + # The downside is slower inference at this single token position, however, this is better than wrong results + if ( + past_key_values + and self.config.rope_scaling + and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1 + ): + past_length = cache_position[0] + if past_length <= self.config.original_max_position_embeddings: + past_key_values = None + + model_inputs = super().prepare_inputs_for_generation( + input_ids=input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + cache_position=cache_position, + position_ids=position_ids, + use_cache=use_cache, + logits_to_keep=logits_to_keep, + **kwargs, + ) + return model_inputs + + class OVBloomForCausalLM(OVModelForCausalLM): # Adapted from transformers.models.bloom.modeling_bloom.BloomForCausalLM.prepare_inputs_for_generation def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): diff --git a/tests/openvino/test_decoder.py b/tests/openvino/test_decoder.py index ba45d393f5..35e67a2731 100644 --- a/tests/openvino/test_decoder.py +++ b/tests/openvino/test_decoder.py @@ -27,6 +27,7 @@ DeepseekOpenVINOConfig, LFM2MoeOpenVINOConfig, LFM2OpenVINOConfig, + Phi3OpenVINOConfig, Qwen3VLOpenVINOConfig, ) from optimum.exporters.openvino.model_patcher import patch_update_causal_mask @@ -78,7 +79,6 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "cohere", "qwen2", "qwen2_moe", - "phi3", "gemma2", "granite", "granitemoe", @@ -86,8 +86,12 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon_mamba") + if is_transformers_version(">=", "4.49"): + SUPPORTED_ARCHITECTURES += ("phi3",) + if is_transformers_version(">=", "4.49") and is_transformers_version("<", "5"): SUPPORTED_SSM_ARCHITECTURES += ("zamba2",) + SUPPORTED_ARCHITECTURES += ("phi3-longrope",) if is_transformers_version(">=", "4.53.0"): SUPPORTED_SSM_ARCHITECTURES += ("granitemoehybrid",) @@ -210,6 +214,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): "pegasus": 2, "qwen": 2, "phi": 2, + "phi3-longrope": 4, "internlm2": 4, "falcon": 2, "falcon-40b": 2, @@ -315,6 +320,8 @@ def test_find_untested_architectures(self): supported_architectures -= {"lfm2"} if is_transformers_version("<", str(LFM2MoeOpenVINOConfig.MIN_TRANSFORMERS_VERSION)): supported_architectures -= {"lfm2_moe"} + if is_transformers_version("<", str(Phi3OpenVINOConfig.MIN_TRANSFORMERS_VERSION)): + supported_architectures -= {"phi3"} # qwen3_vl_text a part of qwen3_vl architecture and is tested in seq2seq group if is_transformers_version(">=", str(Qwen3VLOpenVINOConfig.MIN_TRANSFORMERS_VERSION)): supported_architectures -= {"qwen3_vl_text"} @@ -1029,3 +1036,59 @@ def test_hybrid_model_multi_step_generation(self, model_arch): del transformers_model del ov_model gc.collect() + + def test_phi3_longrope_support(self): + """Test LongRoPE support for Phi3 with inputs > 4096 tokens.""" + if is_transformers_version("<", "4.49"): + self.skipTest("Incompatible transformers version: Phi3 longrope requires transformers>=4.49") + if is_transformers_version(">=", "5"): + self.skipTest( + "Transformers v4 and v5 output different (reference) results for Phi3 longrope. Currently, OpenVINO matches v4 output in both cases." + ) + + set_seed(SEED) + model_id = MODEL_NAMES["phi3-longrope"] + + transformers_model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32) + + # Doublecheck that model has LongRoPE support + original_max_pos = getattr(transformers_model.config, "original_max_position_embeddings", None) + self.assertIsNotNone( + original_max_pos, + f"Model {model_id} does not have original_max_position_embeddings attribute required for LongRoPE", + ) + + set_seed(SEED) + ov_model = OVModelForCausalLM.from_pretrained( + model_id, export=True, ov_config=F32_CONFIG, device=OPENVINO_DEVICE + ) + + # Test 1: input tokens exceed original_max_pos + # Creating model inputs with more than original max position embeddings and enough variation for varied output tokens + tokens = torch.randint(high=transformers_model.config.vocab_size, size=(1, original_max_pos + 50)) + with torch.no_grad(): + transformers_outputs = transformers_model.generate(tokens, max_new_tokens=20) + ov_outputs = ov_model.generate(tokens, max_new_tokens=20) + + self.assertTrue( + torch.equal(transformers_outputs, ov_outputs), + f"OpenVINO and PyTorch outputs do not match for LongRoPE test with inputs > original_max_pos.\n" + f"ov_outputs: {ov_outputs}\ntransformers_outputs: {transformers_outputs}", + ) + + # Test 2: generation tokens exceed original_max_pos + # Creating model inputs with slightly less than original max position embeddings + tokens = torch.randint(high=transformers_model.config.vocab_size, size=(1, original_max_pos - 50)) + with torch.no_grad(): + transformers_outputs = transformers_model.generate(tokens, max_new_tokens=100) + ov_outputs = ov_model.generate(tokens, max_new_tokens=100) + + self.assertTrue( + torch.equal(transformers_outputs, ov_outputs), + f"OpenVINO and PyTorch outputs do not match for LongRoPE test with cumulative context > max_pos.\n" + f"ov_outputs: {ov_outputs}\ntransformers_outputs: {transformers_outputs}", + ) + + del transformers_model + del ov_model + gc.collect() diff --git a/tests/openvino/test_genai.py b/tests/openvino/test_genai.py index 7c6a90c045..970f902aef 100644 --- a/tests/openvino/test_genai.py +++ b/tests/openvino/test_genai.py @@ -145,7 +145,6 @@ class LLMPipelineTestCase(unittest.TestCase): "cohere", "qwen2", "qwen2_moe", - "phi3", "gemma2", "granite", "granitemoe", @@ -166,6 +165,8 @@ class LLMPipelineTestCase(unittest.TestCase): ALL_SUPPORTED_ARCHITECTURES += ("qwen",) if is_transformers_version(">=", "4.48.0"): ALL_SUPPORTED_ARCHITECTURES += ("cohere2",) + if is_transformers_version(">=", "4.49"): + ALL_SUPPORTED_ARCHITECTURES += ("phi3",) if is_transformers_version(">=", "4.50"): ALL_SUPPORTED_ARCHITECTURES += ("gemma3_text",) if is_transformers_version(">=", "4.51.0"): diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 5edbd3a8d5..6445bc4541 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -281,6 +281,7 @@ def _create_tiny_kokoro_model(): "pix2struct": "optimum-intel-internal-testing/pix2struct-tiny-random", "phi": "optimum-intel-internal-testing/tiny-random-PhiForCausalLM", "phi3": "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", + "phi3-longrope": "optimum-intel-internal-testing/tiny-random-phi3-longrope", "phimoe": "optimum-intel-internal-testing/phi-3.5-moe-tiny-random", "phi3_v": "optimum-intel-internal-testing/tiny-random-phi3-vision", "phi4mm": "optimum-intel-internal-testing/tiny-random-phi-4-multimodal",