[OpenVINO] Fix long context inference for Phi3.5 and Phi4

optimum/exporters/openvino/__main__.py

@@ -409,7 +409,7 @@ def main_export(
         # for avoiding confusion we disable remote code for them
         if (
             trust_remote_code
-            and model_type in {"falcon", "mpt", "phi"}
+            and model_type in {"falcon", "mpt", "phi", "phi3"}
             and ("with-past" in task or original_task == "auto")
             and not custom_export_configs
         ):

optimum/exporters/openvino/model_configs.py

@@ -912,7 +912,7 @@ class Phi3OpenVINOConfig(TextDecoderWithPositionIdsOpenVINOConfig):
         MistralDummyPastKeyValuesGenerator,
     ) + TextDecoderOpenVINOConfig.DUMMY_INPUT_GENERATOR_CLASSES
     DUMMY_PKV_GENERATOR_CLASS = MistralDummyPastKeyValuesGenerator
-    MIN_TRANSFORMERS_VERSION = "4.36.0"
+    MIN_TRANSFORMERS_VERSION = "4.49.0"
     _MODEL_PATCHER = Phi3ModelPatcher
 
 

optimum/exporters/openvino/model_patcher.py

@@ -458,12 +458,24 @@ def patch_cos_sin_cached_fp32(model):
 
 
 # Adapted from https://github.com/huggingface/transformers/blob/3c307e380ad07ca16903a39e09a47d532cb782d9/src/transformers/models/phimoe/modular_phimoe.py#L57
-def _longrope_forward(self, x, position_ids=None, layer_type=None):
+def _longrope_forward(self, x, position_ids=None, layer_type=None, **kwargs):
     # _compute_longrope_parameters https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/modeling_rope_utils.py#L391
-    self.config.standardize_rope_params()
-    rope_parameters = (
-        self.config.rope_parameters[layer_type] if layer_type is not None else self.config.rope_parameters
-    )
+    # transformers >= 5 stores RoPE settings under config.rope_parameters; transformers < 5 (e.g. 4.57) stores
+    # them under config.rope_scaling and as plain attributes on the config.
+    if hasattr(self.config, "rope_parameters") and self.config.rope_parameters is not None:
+        rope_parameters = (
+            self.config.rope_parameters[layer_type] if layer_type is not None else self.config.rope_parameters
+        )
+    else:
+        rope_scaling = getattr(self.config, "rope_scaling", None) or {}
+        rope_parameters = dict(rope_scaling)
+        rope_parameters.setdefault("rope_theta", getattr(self.config, "rope_theta", 10000.0))
+        rope_parameters.setdefault(
+            "original_max_position_embeddings",
+            getattr(self.config, "original_max_position_embeddings", self.config.max_position_embeddings),
+        )
+        rope_parameters.setdefault("partial_rotary_factor", getattr(self.config, "partial_rotary_factor", 1.0))
+
     rope_theta = rope_parameters["rope_theta"]
     long_factor = rope_parameters["long_factor"]
     short_factor = rope_parameters["short_factor"]
@@ -474,9 +486,11 @@ def _longrope_forward(self, x, position_ids=None, layer_type=None):
     head_dim = getattr(self.config, "head_dim", self.config.hidden_size // self.config.num_attention_heads)
     dim = int(head_dim * partial_rotary_factor)
 
-    seq_len = torch.max(position_ids) + 1
+    # needed for transformers < v5 for phimoe
+    seq_len = kwargs.get("seq_len", None)
+    _seq_len = seq_len if position_ids is None else torch.max(position_ids) + 1
     # bool tensor to avoid only one path getting traced
-    is_long = seq_len > original_max
+    is_long = _seq_len > original_max
 
     # Compute the inverse frequencies -- scaled based on the target sequence length
     long_factors = torch.tensor(long_factor, dtype=torch.float32, device=x.device)
@@ -498,12 +512,20 @@ def _longrope_forward(self, x, position_ids=None, layer_type=None):
             attention_factor = 1.0 if factor <= 1.0 else math.sqrt(1 + math.log(factor) / math.log(original_max))
         mscale = attention_factor
 
-    # https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/phimoe/modeling_phimoe.py#L116
-    inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
-    position_ids_expanded = position_ids[:, None, :].float()
-    device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
-    with torch.autocast(device_type=device_type, enabled=False):
-        freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+    if is_transformers_version(">=", "5") or position_ids is not None:
+        # https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/phimoe/modeling_phimoe.py#L116
+        inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * mscale
+            sin = emb.sin() * mscale
+    else:
+        # needed for transformers < v5 for phimoe
+        t = torch.arange(_seq_len, device=x.device, dtype=torch.float32)
+        freqs = torch.outer(t, inv_freq)
         emb = torch.cat((freqs, freqs), dim=-1)
         cos = emb.cos() * mscale
         sin = emb.sin() * mscale
@@ -532,12 +554,14 @@ def __enter__(self):
             # non-stateful models on cpu and stateful models on npu
             ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", eager_mask_without_vmap)
 
-        if is_transformers_version(">=", "5"):
-            for module in self._model.modules():
-                rope_type = getattr(module, "rope_type", None)
-                if rope_type == "longrope" and isinstance(getattr(module, "config", None), RotaryEmbeddingConfigMixin):
-                    module._rope_orig_forward = module.forward
-                    module.forward = types.MethodType(_longrope_forward, module)
+        for module in self._model.modules():
+            rope_type = getattr(module, "rope_type", None)
+            if rope_type == "longrope" and (
+                is_transformers_version("<", "5")
+                or isinstance(getattr(module, "config", None), RotaryEmbeddingConfigMixin)
+            ):
+                module._rope_orig_forward = module.forward
+                module.forward = types.MethodType(_longrope_forward, module)
 
     def __exit__(self, exc_type, exc_value, traceback):
         super().__exit__(exc_type, exc_value, traceback)
@@ -550,11 +574,10 @@ def __exit__(self, exc_type, exc_value, traceback):
             ALL_MASK_ATTENTION_FUNCTIONS.register("sdpa", sdpa_mask)
             ALL_MASK_ATTENTION_FUNCTIONS.register("eager", eager_mask)
 
-        if is_transformers_version(">=", "5"):
-            for module in self._model.modules():
-                if hasattr(module, "_rope_orig_forward"):
-                    module.forward = module._rope_orig_forward
-                    del module._rope_orig_forward
+        for module in self._model.modules():
+            if hasattr(module, "_rope_orig_forward"):
+                module.forward = module._rope_orig_forward
+                del module._rope_orig_forward
 
 
 def _mixtral_sparse_moe_block_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -1828,11 +1851,8 @@ class Phi3ModelPatcher(OVDecoderModelPatcher):
     def __enter__(self):
         super().__enter__()
 
-        # currently, long RoPE can not be traced for long context support, disable it for avoid potential accuracy issues
-        if self._model.config.max_position_embeddings != getattr(
-            self._model.config, "original_max_position_embeddings", self._model.config.max_position_embeddings
-        ):
-            self._model.config.max_position_embeddings = self._model.config.original_max_position_embeddings
+        # LongRoPE is handled via _longrope_forward in OVDecoderModelPatcher for both transformers >= 5 and < 5,
+        # so keep the original max_position_embeddings to preserve the correct attention scaling factor.
 
         if is_transformers_version("<", "4.48.0"):
             self._model.model._orig_forward = self._model.model.forward

optimum/intel/openvino/modeling_decoder.py

@@ -898,6 +898,8 @@ def _from_pretrained(
             init_cls = OVBloomForCausalLM
         elif model_type == "gpt_bigcode":
             init_cls = OVGPTBigCodeForCausalLM
+        elif model_type == "phi3":
+            init_cls = OVPhi3ForCausalLM
         elif model_type in SSM_MODELS:
             init_cls = OVModelWithMambaForCausalLM
         else:
@@ -950,6 +952,48 @@ def _from_pretrained(
         return causal_model
 
 
+class OVPhi3ForCausalLM(OVModelForCausalLM):
+    # Adapted from https://github.com/huggingface/transformers/blob/v4.57.0/src/transformers/models/phi3/modeling_phi3.py#L493
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        use_cache=True,
+        logits_to_keep=None,
+        **kwargs,
+    ):
+        # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the
+        # process
+
+        # When the first time input length reached long and short factor switching point, enforce re-compute cache
+        # The downside is slower inference at this single token position, however, this is better than wrong results
+        if (
+            past_key_values
+            and self.config.rope_scaling
+            and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1
+        ):
+            past_length = cache_position[0]
+            if past_length <= self.config.original_max_position_embeddings:
+                past_key_values = None
+
+        model_inputs = super().prepare_inputs_for_generation(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            position_ids=position_ids,
+            use_cache=use_cache,
+            logits_to_keep=logits_to_keep,
+            **kwargs,
+        )
+        return model_inputs
+
+
 class OVBloomForCausalLM(OVModelForCausalLM):
     # Adapted from transformers.models.bloom.modeling_bloom.BloomForCausalLM.prepare_inputs_for_generation
     def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs):

tests/openvino/test_decoder.py

@@ -27,6 +27,7 @@
     DeepseekOpenVINOConfig,
     LFM2MoeOpenVINOConfig,
     LFM2OpenVINOConfig,
+    Phi3OpenVINOConfig,
     Qwen3VLOpenVINOConfig,
 )
 from optimum.exporters.openvino.model_patcher import patch_update_causal_mask
@@ -78,16 +79,19 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "cohere",
         "qwen2",
         "qwen2_moe",
-        "phi3",
         "gemma2",
         "granite",
         "granitemoe",
     )
 
     SUPPORTED_SSM_ARCHITECTURES = ("mamba", "falcon_mamba")
 
+    if is_transformers_version(">=", "4.49"):
+        SUPPORTED_ARCHITECTURES += ("phi3",)
+
     if is_transformers_version(">=", "4.49") and is_transformers_version("<", "5"):
         SUPPORTED_SSM_ARCHITECTURES += ("zamba2",)
+        SUPPORTED_ARCHITECTURES += ("phi3-longrope",)
 
     if is_transformers_version(">=", "4.53.0"):
         SUPPORTED_SSM_ARCHITECTURES += ("granitemoehybrid",)
@@ -210,6 +214,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
         "pegasus": 2,
         "qwen": 2,
         "phi": 2,
+        "phi3-longrope": 4,
         "internlm2": 4,
         "falcon": 2,
         "falcon-40b": 2,
@@ -315,6 +320,8 @@ def test_find_untested_architectures(self):
             supported_architectures -= {"lfm2"}
         if is_transformers_version("<", str(LFM2MoeOpenVINOConfig.MIN_TRANSFORMERS_VERSION)):
             supported_architectures -= {"lfm2_moe"}
+        if is_transformers_version("<", str(Phi3OpenVINOConfig.MIN_TRANSFORMERS_VERSION)):
+            supported_architectures -= {"phi3"}
         # qwen3_vl_text a part of qwen3_vl architecture and is tested in seq2seq group
         if is_transformers_version(">=", str(Qwen3VLOpenVINOConfig.MIN_TRANSFORMERS_VERSION)):
             supported_architectures -= {"qwen3_vl_text"}
@@ -1029,3 +1036,59 @@ def test_hybrid_model_multi_step_generation(self, model_arch):
         del transformers_model
         del ov_model
         gc.collect()
+
+    def test_phi3_longrope_support(self):
+        """Test LongRoPE support for Phi3 with inputs > 4096 tokens."""
+        if is_transformers_version("<", "4.49"):
+            self.skipTest("Incompatible transformers version: Phi3 longrope requires transformers>=4.49")
+        if is_transformers_version(">=", "5"):
+            self.skipTest(
+                "Transformers v4 and v5 output different (reference) results for Phi3 longrope. Currently, OpenVINO matches v4 output in both cases."
+            )
+
+        set_seed(SEED)
+        model_id = MODEL_NAMES["phi3-longrope"]
+
+        transformers_model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)
+
+        # Doublecheck that model has LongRoPE support
+        original_max_pos = getattr(transformers_model.config, "original_max_position_embeddings", None)
+        self.assertIsNotNone(
+            original_max_pos,
+            f"Model {model_id} does not have original_max_position_embeddings attribute required for LongRoPE",
+        )
+
+        set_seed(SEED)
+        ov_model = OVModelForCausalLM.from_pretrained(
+            model_id, export=True, ov_config=F32_CONFIG, device=OPENVINO_DEVICE
+        )
+
+        # Test 1: input tokens exceed original_max_pos
+        # Creating model inputs with more than original max position embeddings and enough variation for varied output tokens
+        tokens = torch.randint(high=transformers_model.config.vocab_size, size=(1, original_max_pos + 50))
+        with torch.no_grad():
+            transformers_outputs = transformers_model.generate(tokens, max_new_tokens=20)
+            ov_outputs = ov_model.generate(tokens, max_new_tokens=20)
+
+        self.assertTrue(
+            torch.equal(transformers_outputs, ov_outputs),
+            f"OpenVINO and PyTorch outputs do not match for LongRoPE test with inputs > original_max_pos.\n"
+            f"ov_outputs: {ov_outputs}\ntransformers_outputs: {transformers_outputs}",
+        )
+
+        # Test 2: generation tokens exceed original_max_pos
+        # Creating model inputs with slightly less than original max position embeddings
+        tokens = torch.randint(high=transformers_model.config.vocab_size, size=(1, original_max_pos - 50))
+        with torch.no_grad():
+            transformers_outputs = transformers_model.generate(tokens, max_new_tokens=100)
+            ov_outputs = ov_model.generate(tokens, max_new_tokens=100)
+
+        self.assertTrue(
+            torch.equal(transformers_outputs, ov_outputs),
+            f"OpenVINO and PyTorch outputs do not match for LongRoPE test with cumulative context > max_pos.\n"
+            f"ov_outputs: {ov_outputs}\ntransformers_outputs: {transformers_outputs}",
+        )
+
+        del transformers_model
+        del ov_model
+        gc.collect()

tests/openvino/test_genai.py

@@ -145,7 +145,6 @@ class LLMPipelineTestCase(unittest.TestCase):
         "cohere",
         "qwen2",
         "qwen2_moe",
-        "phi3",
         "gemma2",
         "granite",
         "granitemoe",
@@ -166,6 +165,8 @@ class LLMPipelineTestCase(unittest.TestCase):
         ALL_SUPPORTED_ARCHITECTURES += ("qwen",)
     if is_transformers_version(">=", "4.48.0"):
         ALL_SUPPORTED_ARCHITECTURES += ("cohere2",)
+    if is_transformers_version(">=", "4.49"):
+        ALL_SUPPORTED_ARCHITECTURES += ("phi3",)
     if is_transformers_version(">=", "4.50"):
         ALL_SUPPORTED_ARCHITECTURES += ("gemma3_text",)
     if is_transformers_version(">=", "4.51.0"):

tests/openvino/utils_tests.py

@@ -281,6 +281,7 @@ def _create_tiny_kokoro_model():
     "pix2struct": "optimum-intel-internal-testing/pix2struct-tiny-random",
     "phi": "optimum-intel-internal-testing/tiny-random-PhiForCausalLM",
     "phi3": "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM",
+    "phi3-longrope": "optimum-intel-internal-testing/tiny-random-phi3-longrope",
     "phimoe": "optimum-intel-internal-testing/phi-3.5-moe-tiny-random",
     "phi3_v": "optimum-intel-internal-testing/tiny-random-phi3-vision",
     "phi4mm": "optimum-intel-internal-testing/tiny-random-phi-4-multimodal",