[model] Add support for LLaVA-OneVision-1.5 model (#54)

Author: randydl

src/mcore_bridge/config/parser.py

@@ -118,7 +118,7 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]:
     window_size = res.pop('window_size', None)
     rope_scaling = res.get('rope_scaling') or {}
     if llm_model_type in {'qwen3', 'qwen3_moe', 'qwen3_next'} or hf_model_type in {
-            'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe'
+            'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5'
     }:
         res['qk_layernorm'] = True
     if llm_model_type in {'qwen2_moe', 'qwen3_moe', 'qwen3_next'

src/mcore_bridge/model/constant.py

@@ -30,6 +30,8 @@ class MLLMModelType:
 
     kimi_k25 = 'kimi_k25'
 
+    llava_onevision1_5 = 'llava_onevision1_5'
+
 
 class ModelType(LLMModelType, MLLMModelType):
     pass

src/mcore_bridge/model/mm_gpts/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
-from . import glm, internvl, kimi_vl, llama4, qwen, qwen3_5, qwen3_5_gdn, qwen3_omni, qwen3_vl
+from . import glm, internvl, kimi_vl, llama4, llava, qwen, qwen3_5, qwen3_5_gdn, qwen3_omni, qwen3_vl

src/mcore_bridge/model/mm_gpts/llava.py

@@ -0,0 +1,33 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from transformers import PretrainedConfig
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from mcore_bridge.bridge import GPTBridge
+
+from ..constant import ModelType
+from ..register import ModelMeta, register_model
+from .utils import HuggingFaceVit
+
+
+class LLavaOneVision1_5Vit(HuggingFaceVit):
+    module_mapping = {'visual': 'visual'}
+    _vision_tower = ['visual']
+    _aligner = ['visual.merger']
+
+    def prepare_model(self, hf_config: PretrainedConfig):
+        VisualModel = get_class_from_dynamic_module(
+            'modeling_llavaonevision1_5.RiceTransformerPretrainedModel',
+            hf_config.name_or_path)
+        self.visual = VisualModel._from_config(hf_config.vision_config)
+
+    def get_inputs_embeds(self, inputs_embeds, **kwargs):
+        return self._hf_get_inputs_embeds(inputs_embeds, kwargs, self.visual, self.hf_config)
+
+
+register_model(
+    ModelMeta(
+        ModelType.llava_onevision1_5,
+        ['llava_onevision1_5'],
+        bridge_cls=GPTBridge,
+        visual_cls=LLavaOneVision1_5Vit,
+    ))

tests/test_mllm.py

@@ -108,6 +108,10 @@ def test_qwen3_5():
     # _test_model('Qwen/Qwen3.5-27B')
 
 
+def test_llava_onevision1_5():
+    _test_model('lmms-lab/LLaVA-OneVision-1.5-4B-Instruct')
+
+
 if __name__ == '__main__':
     # test_qwen2_5_vl()
     # test_qwen2_vl()
@@ -126,4 +130,5 @@ def test_qwen3_5():
     # test_qwen3_vl_moe()
     # test_qwen3_omni()
     # test_llama4()
-    test_qwen3_5()
+    # test_qwen3_5()
+    test_llava_onevision1_5()