Skip to content

Commit d513dfe

Browse files
xufang-lisaCopilot
xufang-lisa
and
Copilot
authored
support videochat (#1637)
* support videochat_flash_qwen * fix error * remove unused function * fix hidden_size for vision projectio * add preprocess_inputs * set default quantization config for videochat model * add rotary_pos_embed to vision_embedding * update vision projection input name * use mm_hidden_size as embed_dim * add check for videochat * Add pipeline for VideoChat * support text only * remove unused code * add videochat test * add test dependencies * fix style check issue * fix style check issue * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * apply code review comments * fix code style * fix fail in 4.45 * update _DEFAULT_4BIT_WQ_CONFIGS * add test to compare inference results with transformers * apply comments * fix test fail in transformers 4.45 * pad frame number to a multiple of 4 * fix file locking issue on windows * apply comments * update comments * fix code style * add default image_preprocess * apply comments * fix code style * update tests * update tests * remove NotImplemented exceptions * fix code style * test videochat export when transformers>=4.49 * fix code style * use torchvision instead of transformers.image_transforms * remove blank line * add patcher comments and remove unnecessary patcher * add comments * apply comments * remove fallback defaults and read directly from config * apply copilot comments * add test for quantization * fix test case * read image_size and patch_size from model * read image_mean and image_std from model and write to config.json * follow the existing code style * prepare modalities in preprocess_inputs * remove unnecessary changes * fix code format * remove the get_gext_embeddings function wrapper * avoid copying codes by loading original model * update test condition for videochat * move parameters assignments into model config class * remove unnecessary changes --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent c877c15 commit d513dfe

13 files changed

Lines changed: 807 additions & 9 deletions

File tree

‎docs/source/openvino/models.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ Here is the list of the supported architectures :
160160
- TrOCR
161161
- UniSpeech
162162
- UniSpeech-SAT
163+
- VideoChat-Flash-Qwen
163164
- Vision Encoder Decoder
164165
- ViT
165166
- Wav2Vec2

‎optimum/exporters/openvino/model_configs.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@
213213
Qwen3VLVisionEmbMergerPatcher,
214214
QwenModelPatcher,
215215
SanaTextEncoderModelPatcher,
216+
VideoChatFlashQwenVisionEmbeddingModelPatcher,
216217
XverseModelPatcher,
217218
Zamba2ModelPatcher,
218219
_get_model_attribute,
@@ -5764,6 +5765,218 @@ class SiglipTextOpenVINOConfig(SiglipTextOnnxConfig):
57645765
pass
57655766

57665767

5768+
class DummyVideoChatFlashQwenInputGenerator(DummyVisionInputGenerator):
5769+
SUPPORTED_INPUT_NAMES = ("hidden_states", "rotary_pos_emb")
5770+
5771+
def __init__(
5772+
self,
5773+
task: str,
5774+
normalized_config: NormalizedVisionConfig,
5775+
batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"],
5776+
num_channels: int = DEFAULT_DUMMY_SHAPES["num_channels"],
5777+
width: int = DEFAULT_DUMMY_SHAPES["width"],
5778+
height: int = DEFAULT_DUMMY_SHAPES["height"],
5779+
visual_seq_length: int = DEFAULT_DUMMY_SHAPES["visual_seq_length"],
5780+
**kwargs,
5781+
):
5782+
super().__init__(task, normalized_config, batch_size, num_channels, width, height, visual_seq_length, **kwargs)
5783+
self.num_frames = normalized_config.config.mm_local_num_frames
5784+
self.embed_dim = normalized_config.config.mm_hidden_size
5785+
self.height = normalized_config.config.image_size
5786+
self.width = normalized_config.config.image_size
5787+
self.image_size = (self.height, self.width)
5788+
self.patch_size = normalized_config.config.patch_size
5789+
5790+
def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
5791+
if input_name == "hidden_states":
5792+
return self.random_float_tensor(
5793+
shape=[
5794+
self.batch_size,
5795+
self.num_channels,
5796+
self.num_frames,
5797+
self.height,
5798+
self.width,
5799+
],
5800+
framework=framework,
5801+
dtype=float_dtype,
5802+
)
5803+
elif input_name == "rotary_pos_emb":
5804+
grid_h, grid_w = self.height // self.patch_size, self.width // self.patch_size
5805+
grid_t = self.num_frames
5806+
# Source: https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B/blob/main/vision_tower_builder.py#L523
5807+
# The first dimension of rotary_pos_emb is fixed to 1 in the original model.
5808+
# And the second dimension is the total number of tokens for all frames, which is calculated as grid_h * grid_w * grid_t plus 1 for the cls token.
5809+
return self.random_float_tensor(
5810+
[1, 1 + grid_h * grid_t * grid_w, self.embed_dim], framework=framework, dtype=float_dtype
5811+
)
5812+
5813+
5814+
class DummyVideoChatFlashQwenProjectorInputGenerator(DummyInputGenerator):
5815+
SUPPORTED_INPUT_NAMES = ["input"]
5816+
5817+
def __init__(
5818+
self,
5819+
task: str,
5820+
normalized_config: NormalizedTextConfig,
5821+
batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"],
5822+
random_batch_size_range: Optional[Tuple[int, int]] = None,
5823+
**kwargs,
5824+
):
5825+
self.task = task
5826+
self.batch_size = batch_size
5827+
self.hidden_size = normalized_config.config.mm_hidden_size
5828+
self.num_patches = normalized_config.config.mm_projector_num_tome_tokens
5829+
self.normalized_config = normalized_config
5830+
5831+
def generate(
5832+
self,
5833+
input_name: str,
5834+
framework: str = "pt",
5835+
int_dtype: str = "int64",
5836+
float_dtype: str = "fp32",
5837+
):
5838+
shape = [self.batch_size, self.num_patches, self.hidden_size]
5839+
return self.random_float_tensor(shape, framework=framework, dtype=float_dtype)
5840+
5841+
5842+
class VideoChatFlashQwenProjectorOpenVINOConfig(OnnxConfig):
5843+
DUMMY_INPUT_GENERATOR_CLASSES = (DummyVideoChatFlashQwenProjectorInputGenerator,)
5844+
NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig
5845+
5846+
@property
5847+
def inputs(self) -> Dict[str, Dict[int, str]]:
5848+
return {"input": {0: "batch_size", 1: "num_patches", 2: "hidden_size"}}
5849+
5850+
5851+
class VideoChatFlashQwenConfigBehavior(str, enum.Enum):
5852+
LANGUAGE = "language"
5853+
VISION_EMBEDDINGS = "vision_embeddings"
5854+
VISION_PROJECTION = "vision_projection"
5855+
TEXT_EMBEDDINGS = "text_embeddings"
5856+
5857+
5858+
@register_in_tasks_manager("videochat_flash_qwen", *["image-text-to-text"], library_name="transformers")
5859+
class VideoChatFlashQwenOpenVINOConfig(BaseVLMOpenVINOConfig):
5860+
MIN_TRANSFORMERS_VERSION = "4.49.0"
5861+
MAX_TRANSFORMERS_VERSION = "4.57.6"
5862+
SUPPORTED_BEHAVIORS = [model_type.value for model_type in VideoChatFlashQwenConfigBehavior]
5863+
DUMMY_INPUT_GENERATOR_CLASSES = (DummyVideoChatFlashQwenInputGenerator,)
5864+
5865+
def __init__(
5866+
self,
5867+
config: "PretrainedConfig",
5868+
task: str = "feature-extraction",
5869+
int_dtype: str = "int64",
5870+
float_dtype: str = "fp32",
5871+
behavior: VideoChatFlashQwenConfigBehavior = VideoChatFlashQwenConfigBehavior.VISION_EMBEDDINGS,
5872+
preprocessors: Optional[List[Any]] = None,
5873+
**kwargs,
5874+
):
5875+
super().__init__(
5876+
config=config,
5877+
task=task,
5878+
int_dtype=int_dtype,
5879+
float_dtype=float_dtype,
5880+
behavior=behavior,
5881+
preprocessors=preprocessors,
5882+
)
5883+
self._orig_config = config
5884+
self._model_config_prepared = False
5885+
5886+
@property
5887+
def inputs(self) -> Dict[str, Dict[int, str]]:
5888+
if not self._behavior == VideoChatFlashQwenConfigBehavior.VISION_EMBEDDINGS:
5889+
return {}
5890+
return {
5891+
"hidden_states": {0: "batch_size", 2: "num_frames", 3: "height", 4: "width"},
5892+
# rotary_pos_emb has a fixed leading dimension of 1 in the dummy generator,
5893+
# so we do not associate axis 0 with batch_size and keep only dynamic axes here.
5894+
"rotary_pos_emb": {1: "num_tokens", 2: "hidden_size"},
5895+
}
5896+
5897+
def with_behavior(
5898+
self,
5899+
behavior: Union[str, VideoChatFlashQwenConfigBehavior],
5900+
):
5901+
"""
5902+
Creates a config for different behaviour.
5903+
5904+
Args:
5905+
behavior ([`ConfigBehavior`]):
5906+
The behavior to use for the new instance.
5907+
"""
5908+
if isinstance(behavior, str) and not isinstance(behavior, VideoChatFlashQwenConfigBehavior):
5909+
behavior = VideoChatFlashQwenConfigBehavior(behavior)
5910+
5911+
if behavior == VideoChatFlashQwenConfigBehavior.VISION_PROJECTION:
5912+
export_config = VideoChatFlashQwenProjectorOpenVINOConfig(
5913+
self._orig_config,
5914+
task="feature-extraction",
5915+
int_dtype=self.int_dtype,
5916+
float_dtype=self.float_dtype,
5917+
)
5918+
return export_config
5919+
5920+
if behavior == VideoChatFlashQwenConfigBehavior.TEXT_EMBEDDINGS:
5921+
return get_vlm_text_embeddings_config("qwen2", self._orig_config, self.int_dtype, self.float_dtype)
5922+
5923+
if behavior == VideoChatFlashQwenConfigBehavior.LANGUAGE:
5924+
return get_vlm_text_generation_config("qwen2", self._orig_config, self.int_dtype, self.float_dtype)
5925+
5926+
if behavior == VideoChatFlashQwenConfigBehavior.VISION_EMBEDDINGS:
5927+
return self.__class__(
5928+
self._orig_config,
5929+
task=self.task,
5930+
int_dtype=self.int_dtype,
5931+
float_dtype=self.float_dtype,
5932+
behavior=behavior,
5933+
preprocessors=self._preprocessors,
5934+
)
5935+
5936+
def get_model_for_behavior(self, model, behavior: Union[str, VideoChatFlashQwenConfigBehavior]):
5937+
if isinstance(behavior, str) and not isinstance(behavior, VideoChatFlashQwenConfigBehavior):
5938+
behavior = VideoChatFlashQwenConfigBehavior(behavior)
5939+
5940+
if not self._model_config_prepared:
5941+
vision_tower = model.get_vision_tower()
5942+
model.config.mm_num_attention_heads = vision_tower.config.num_attention_heads
5943+
# num_tome_tokens=64 comes from the upstream projector_type "tome16_mlp_hd64", which uses a fixed 64-token output.
5944+
# Source: https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B/blob/main/mm_projector_builder.py#L135-L146
5945+
model.config.mm_projector_num_tome_tokens = 64
5946+
model.config.patch_size = vision_tower.config.patch_size
5947+
model.config.image_size = vision_tower.config.image_size
5948+
model.config.image_mean = vision_tower.image_processor.image_mean
5949+
model.config.image_std = vision_tower.image_processor.image_std
5950+
self._model_config_prepared = True
5951+
5952+
if behavior == VideoChatFlashQwenConfigBehavior.VISION_PROJECTION:
5953+
vision_projector = model.get_model().mm_projector.mlp
5954+
vision_projector.config = model.config
5955+
return vision_projector
5956+
5957+
if behavior == VideoChatFlashQwenConfigBehavior.VISION_EMBEDDINGS:
5958+
vision_tower = model.get_vision_tower().vision_tower
5959+
vision_tower.config = model.config
5960+
return vision_tower
5961+
5962+
if behavior == VideoChatFlashQwenConfigBehavior.TEXT_EMBEDDINGS:
5963+
text_embedding = model.get_input_embeddings()
5964+
text_embedding.config = model.config
5965+
return text_embedding
5966+
5967+
if behavior == VideoChatFlashQwenConfigBehavior.LANGUAGE:
5968+
model.model.llm_compress_layer_list = []
5969+
return model
5970+
5971+
def patch_model_for_export(self, model: PreTrainedModel, model_kwargs: Optional[Dict[str, Any]] = None):
5972+
model_kwargs = model_kwargs or {}
5973+
5974+
if self._behavior == VideoChatFlashQwenConfigBehavior.VISION_EMBEDDINGS:
5975+
return VideoChatFlashQwenVisionEmbeddingModelPatcher(self, model, model_kwargs)
5976+
5977+
return super().patch_model_for_export(model, model_kwargs)
5978+
5979+
57675980
@register_in_tasks_manager(
57685981
"hunyuan_v1_dense",
57695982
*[

‎optimum/exporters/openvino/model_patcher.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8198,6 +8198,44 @@ def __exit__(self, exc_type, exc_value, traceback):
81988198
del afmoe_moe.down_projs, afmoe_moe.gate_projs, afmoe_moe.up_projs
81998199

82008200

8201+
class VideoChatFlashQwenVisionEmbeddingModelPatcher(ModelPatcher):
8202+
def __init__(
8203+
self,
8204+
config: "OnnxConfig",
8205+
model: "PreTrainedModel",
8206+
model_kwargs: Dict[str, Any] = None,
8207+
):
8208+
model.__orig_forward = model.forward
8209+
8210+
# Modified from https://huggingface.co/OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B/blob/main/vision_tower_builder.py#L618-L675
8211+
# Export keeps only one traced branch, while this model needs both image and video paths.
8212+
# We expose rotary_pos_emb as an input (instead of internal pos_embed/image_pos_embed) so the caller
8213+
# decides which positional embedding to pass for image vs video.
8214+
# This also simplifies internal logic that is not needed for the export path (like residual is always None and x_vis_only is always True in original model).
8215+
def forward_wrap(self, hidden_states, rotary_pos_emb):
8216+
hidden_states = self.patch_embed(hidden_states.type(self.dtype))
8217+
B, T, L, C = hidden_states.shape # T: temporal; L: spatial
8218+
hidden_states = hidden_states.view([B, T * L, C])
8219+
8220+
# append cls token
8221+
cls_tokens = self.cls_token.expand(B, -1, -1)
8222+
hidden_states = torch.cat((cls_tokens, hidden_states), dim=1)
8223+
hidden_states = hidden_states + rotary_pos_emb
8224+
hidden_states = hidden_states.reshape(B, -1, C)
8225+
8226+
for idx, blk in enumerate(self.blocks):
8227+
hidden_states = blk(hidden_states, residual=None)
8228+
8229+
return hidden_states
8230+
8231+
model.forward = types.MethodType(forward_wrap, model)
8232+
super().__init__(config, model, model_kwargs)
8233+
8234+
def __exit__(self, exc_type, exc_value, traceback):
8235+
super().__exit__(exc_type, exc_value, traceback)
8236+
self._model.forward = self._model.__orig_forward
8237+
8238+
82018239
# adopted from https://github.com/huggingface/transformers/blob/v4.57.6/src/transformers/models/llama/modeling_llama.py#L197
82028240
class LlamaEagle3Attention(LlamaAttention):
82038241
"""

‎optimum/exporters/openvino/utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,7 @@ def get_submodels(model):
306306
"phi4_multimodal",
307307
"llama4",
308308
"minicpmo",
309+
"videochat_flash_qwen",
309310
]
310311

311312
SSM_MODELS = [

‎optimum/intel/openvino/configuration.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,18 @@ class OVQuantizationMethod(str, Enum):
428428
"weight_only": True,
429429
},
430430
},
431+
"OpenGVLab/VideoChat-Flash-Qwen2_5-7B_InternVideo2-1B": {
432+
"quantization_configs": {
433+
"lm_model": {
434+
"bits": 4,
435+
"sym": False,
436+
"group_size": 128,
437+
"ratio": 1.0,
438+
},
439+
"text_embeddings_model": {"bits": 8, "sym": True, "weight_only": True},
440+
"vision_embeddings_model": {"bits": 8, "sym": True, "weight_only": True},
441+
},
442+
},
431443
"qnguyen3/nanoLLaVA": {
432444
"bits": 4,
433445
"sym": False,

0 commit comments

Comments
 (0)