Skip to content

[OpenVINO] fix: Respect trust_remote_code in export and improve safety for multimodal configs FIXES #1668 #1673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MissLostCodes wants to merge 14 commits into
base: main
Choose a base branch
from
Open

[OpenVINO] fix: Respect trust_remote_code in export and improve safety for multimodal configs FIXES #1668 #1673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
05536a7
Fix trust_remote_code propagation in OpenVINO export configs
MissLostCodes Apr 2, 2026
8954d32
remove warnings as they are never reached
MissLostCodes Apr 2, 2026
c42bf7f
add trust-remote-code warning
MissLostCodes Apr 2, 2026
12fd48e
added Selective propagation of trust_remote_code
MissLostCodes Apr 3, 2026
ec03941
Format code with Black
MissLostCodes Apr 13, 2026
6b79aac
add models to remote models list
MissLostCodes Apr 13, 2026
e74f020
propogate trc in llmpipeline
MissLostCodes Apr 13, 2026
6890261
oops wrong move
MissLostCodes Apr 14, 2026
b20b34f
DEBUG: force for xglm
MissLostCodes Apr 14, 2026
42695cf
DEBUG: test proves it not due to trust-remote-code , xglm still fails
MissLostCodes Apr 14, 2026
0fbf8ee
t-r-c in with_behaviour for vision embedding- copilot suggestion applied
MissLostCodes Apr 22, 2026
07e3dd4
Clean up
MissLostCodes Apr 28, 2026
aad0ac3
Added tests to ensure trust_remote_code is required for multimodal Op…
MissLostCodes Apr 29, 2026
91359e7
ruff code style
MissLostCodes Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions optimum/exporters/openvino/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,13 @@ def infer_task(
)

if library_name == "transformers":
if not trust_remote_code:
logger.warning(
"This model may require executing custom code from its repository. "
"For security reasons, this is disabled by default. "
"Please review the source and rerun with `--trust-remote-code` if needed."
)

config = AutoConfig.from_pretrained(
model_name_or_path,
subfolder=subfolder,
Expand Down
38 changes: 30 additions & 8 deletions optimum/exporters/openvino/convert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@
set_simplified_chat_template,
)


logger = logging.getLogger(__name__)

if is_torch_available():
Expand Down Expand Up @@ -684,6 +683,7 @@ def export_from_model(
_variant="default",
exporter="openvino",
stateful=stateful,
trust_remote_code=trust_remote_code,
)
logging.disable(logging.NOTSET)

Expand Down Expand Up @@ -916,27 +916,29 @@ def _get_multi_modal_submodels_and_export_configs(
preprocessors: Optional[List[Any]] = None,
model_kwargs: Optional[Dict] = None,
stateful: bool = True,
trust_remote_code: bool = False,
):
models_for_export = {}
stateful_parts = []

model_type = model.config.model_type
base_model_type = model.config.model_type
export_model_type = getattr(model.config, "export_model_type", None) or base_model_type

if model_type == "internvl_chat" and preprocessors is not None:
if base_model_type == "internvl_chat" and preprocessors is not None:
model.config.img_context_token_id = preprocessors[0].convert_tokens_to_ids("<IMG_CONTEXT>")

if model_type == "phi3_v":
if base_model_type == "phi3_v":
model.config.glb_GN = model.model.vision_embed_tokens.glb_GN.tolist()
model.config.sub_GN = model.model.vision_embed_tokens.sub_GN.tolist()

if model_type == "phi4mm":
if base_model_type == "phi4mm":
model.config.glb_GN = model.model.embed_tokens_extend.image_embed.glb_GN.tolist()
model.config.sub_GN = model.model.embed_tokens_extend.image_embed.sub_GN.tolist()
model.config.num_img_tokens = model.model.embed_tokens_extend.image_embed.num_img_tokens
model.config.hd_transform_order = model.model.embed_tokens_extend.image_embed.hd_transform_order
if model.config.img_processor is None:
model.config.img_processor = model.model.embed_tokens_extend.image_embed.img_processor.config.to_dict()
if model_type == "phi4_multimodal":
if base_model_type == "phi4_multimodal":
model.config.glb_GN = model.model.embed_tokens_extend.image_embed.global_img_feature_extensor.tolist()
model.config.sub_GN = model.model.embed_tokens_extend.image_embed.sub_img_feature_extensor.tolist()
model.config.num_img_tokens = model.model.embed_tokens_extend.image_embed.num_img_tokens
Expand All @@ -949,8 +951,19 @@ def _get_multi_modal_submodels_and_export_configs(
main_config_cls = TasksManager.get_exporter_config_constructor(
model=model, task=task, exporter="openvino", library_name=library_name
)

config_kwargs = {
"int_dtype": int_dtype,
"float_dtype": float_dtype,
"preprocessors": preprocessors,
}

if export_model_type in {"llava-qwen2", "phi3_v"}:
config_kwargs["trust_remote_code"] = trust_remote_code

Comment on lines +961 to +963
Copy link
Copy Markdown
Collaborator

@popovaan popovaan Apr 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is trust_remote_code flag propagated here only for "llava-qwen2" and "phi3_v"? Maybe we need to do it unconditionally?

Copy link
Copy Markdown
Contributor Author

@MissLostCodes MissLostCodes Apr 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

other models do require trust_remote_code at the Transformers level but their corresponding OpenVINO config classes do not accept this parameter yet . So the condition is required

propagating trust_remote_code unconditionally, leads to :

TypeError: Qwen2VLOpenVINOConfig.__init__() got an unexpected keyword argument 'trust_remote_code'  
TypeError: InternVLChatOpenVINOConfig.__init__() got an unexpected keyword argument 'trust_remote_code'  
TypeError: MiniCPMVOpenVINOConfig.__init__() got an unexpected keyword argument 'trust_remote_code'

main_config = main_config_cls(
model.config, int_dtype=int_dtype, float_dtype=float_dtype, preprocessors=preprocessors
model.config,
**config_kwargs,
)
for behavior in main_config.SUPPORTED_BEHAVIORS:
model_id = f"{behavior}_model"
Expand All @@ -976,14 +989,23 @@ def _get_submodels_and_export_configs(
model_kwargs: Optional[Dict] = None,
exporter: str = "openvino",
stateful: bool = False,
trust_remote_code: bool = False,
):
if (
not custom_architecture
and library_name == "transformers"
and model.config.model_type in MULTI_MODAL_TEXT_GENERATION_MODELS
):
return _get_multi_modal_submodels_and_export_configs(
model, task, library_name, int_dtype, float_dtype, preprocessors, model_kwargs, stateful
model,
task,
library_name,
int_dtype,
float_dtype,
preprocessors,
model_kwargs,
stateful,
trust_remote_code=trust_remote_code,
)
elif not custom_architecture and library_name == "transformers" and model.config.model_type == "speecht5":
return _get_speecht5_tss_model_for_export(
Expand Down
14 changes: 11 additions & 3 deletions optimum/exporters/openvino/model_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2183,11 +2183,14 @@ def __init__(
behavior: VLMConfigBehavior = VLMConfigBehavior.VISION_EMBEDDINGS,
preprocessors: Optional[List[Any]] = None,
use_past: bool = False,
trust_remote_code: bool = False,
**kwargs,
):
self._behavior = behavior
self._orig_config = config
if self._behavior == VLMConfigBehavior.VISION_EMBEDDINGS:
config = AutoConfig.from_pretrained(config.mm_vision_tower, trust_remote_code=True)
self._trust_remote_code = trust_remote_code
if self._behavior == VLMConfigBehavior.VISION_EMBEDDINGS and hasattr(config, "mm_vision_tower"):
config = AutoConfig.from_pretrained(config.mm_vision_tower, trust_remote_code=self._trust_remote_code)
Comment thread
MissLostCodes marked this conversation as resolved.
if hasattr(config, "vision_config"):
config = config.vision_config
super().__init__(
Expand Down Expand Up @@ -2256,6 +2259,7 @@ def with_behavior(
float_dtype=self.float_dtype,
behavior=behavior,
preprocessors=self._preprocessors,
trust_remote_code=self._trust_remote_code,
)

def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[Dict[str, Any]] = None):
Expand Down Expand Up @@ -3066,6 +3070,8 @@ def __init__(
float_dtype: str = "fp32",
behavior: Phi3VisionConfigBehavior = Phi3VisionConfigBehavior.VISION_EMBEDDINGS,
preprocessors: Optional[List[Any]] = None,
trust_remote_code: bool = False,
**kwargs,
):
super().__init__(
config=config,
Expand All @@ -3076,9 +3082,10 @@ def __init__(
)
self._behavior = behavior
self._orig_config = config
self._trust_remote_code = trust_remote_code
if self._behavior == Phi3VisionConfigBehavior.VISION_EMBEDDINGS and hasattr(config, "img_processor"):
self._config = AutoConfig.from_pretrained(
config.img_processor["model_name"], trust_remote_code=True
config.img_processor["model_name"], trust_remote_code=self._trust_remote_code
Comment thread
MissLostCodes marked this conversation as resolved.
).vision_config
self._normalized_config = self.NORMALIZED_CONFIG_CLASS(self._config)
self.DUMMY_INPUT_GENERATOR_CLASSES = (DummyVisionInputGenerator,)
Expand Down Expand Up @@ -3127,6 +3134,7 @@ def with_behavior(
float_dtype=self.float_dtype,
behavior=behavior,
preprocessors=self._preprocessors,
trust_remote_code=self._trust_remote_code,
)
if behavior == Phi3VisionConfigBehavior.VISION_PROJECTION:
return self.__class__(
Expand Down
25 changes: 24 additions & 1 deletion tests/openvino/test_export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
from optimum.utils import logging
from optimum.utils.save_utils import maybe_load_preprocessors


logger = logging.get_logger()


Expand Down Expand Up @@ -125,6 +124,7 @@ class ExportModelTest(unittest.TestCase):
SUPPORTED_ARCHITECTURES.update({"qwen3": OVModelForFeatureExtraction})

GENERATIVE_MODELS = ("pix2struct", "t5", "bart", "gpt2", "whisper", "llava", "speecht5")
OV_MULTIMODAL_REMOTE_CODE_MODELS = ("llava-qwen2", "phi3_v")

def _openvino_export(
self,
Expand Down Expand Up @@ -334,6 +334,29 @@ def test_compare_openvino_onnx_supported_architectures(self):
if len(only_onnx) > 0:
logger.warning(f"The following architectures export {only_onnx} is supported by ONNX but not OpenVINO")

@parameterized.expand(OV_MULTIMODAL_REMOTE_CODE_MODELS)
Copy link
Copy Markdown
Collaborator

@rkazants rkazants May 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we have a list of all trust remote code models. May be we can parameterize this test using this list.

Copy link
Copy Markdown
Contributor Author

@MissLostCodes MissLostCodes May 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the models in that list donot support trust remote code propogation at openvino level , they handle it at hugging face level so running this test over all those models will throw TypeError for unexpected argument and fail these tests .

def test_export_requires_trust_remote_code_for_multimodal_models(self, model_type):

model = MODEL_NAMES[model_type]
task = "image-text-to-text"
with TemporaryDirectory() as tmpdirname:
with self.assertRaises(ValueError) as ctx:
main_export(
model_name_or_path=model,
task=task,
output=Path(tmpdirname),
trust_remote_code=False,
)

self.assertIn("trust_remote_code", str(ctx.exception))

main_export(
model_name_or_path=model,
task=task,
output=Path(tmpdirname),
trust_remote_code=True,
)


class CustomExportModelTest(unittest.TestCase):
def test_custom_export_config_model(self):
Expand Down
Loading