Skip to content

Commit cb299e6

Browse files
committed
Update(MTMDChatHandler): add chunk type helpers
- Add small helper methods `_is_text_chunk`/`_is_image_chunk`/`_is_audio_chunk` for checking MTMD text, image, and audio chunk type enum values. - This keeps MTMD prompt processing easier to read and avoids repeating direct enum comparisons when building token spans for text and media chunks. Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 971ee38 commit cb299e6

1 file changed

Lines changed: 25 additions & 11 deletions

File tree

llama_cpp/llama_chat_format.py

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3341,6 +3341,26 @@ def _create_bitmap_from_bytes(self, media_bytes: bytes):
33413341

33423342
return wrapper.bitmap, wrapper.video_ctx
33433343

3344+
def _is_text_chunk(self, chunk_type: int) -> bool:
3345+
"""Return True if `chunk_type` is the MTMD text chunk type enum value."""
3346+
return (
3347+
chunk_type
3348+
== self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT
3349+
)
3350+
3351+
def _is_image_chunk(self, chunk_type: int) -> bool:
3352+
"""Return True if `chunk_type` is the MTMD image chunk type enum value."""
3353+
return (
3354+
chunk_type
3355+
== self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE
3356+
)
3357+
3358+
def _is_audio_chunk(self, chunk_type: int) -> bool:
3359+
"""Return True if `chunk_type` is the MTMD audio chunk type enum value."""
3360+
return (
3361+
chunk_type
3362+
== self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO
3363+
)
33443364

33453365
def _process_mtmd_prompt(
33463366
self,
@@ -3480,7 +3500,7 @@ def _create_bitmap_func(idx: int, item: dict):
34803500
if chunk is None: continue
34813501
chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk)
34823502

3483-
if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT:
3503+
if self._is_text_chunk(chunk_type):
34843504
# Extract standard text token IDs
34853505
n_tokens_out = ctypes.c_size_t()
34863506
tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk, ctypes.byref(n_tokens_out))
@@ -3489,10 +3509,7 @@ def _create_bitmap_func(idx: int, item: dict):
34893509
chunk_token_spans.append((current_idx, current_idx + len(tokens), chunk, chunk_type, None))
34903510
full_prompt_ids.extend(tokens)
34913511
current_idx += len(tokens)
3492-
elif chunk_type in [
3493-
self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE,
3494-
self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO
3495-
]:
3512+
elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type):
34963513
# Extract media properties
34973514
# Note(JamePeng):
34983515
# The M-RoPE model is based on `n_pos` instead of `n_tokens` (of course, there's no difference in non-M-RoPE models).
@@ -3673,7 +3690,7 @@ def __call__(
36733690
if end_idx <= n_past:
36743691
continue
36753692

3676-
if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT:
3693+
if self._is_text_chunk(chunk_type):
36773694
unprocessed_start = max(start_idx, n_past) - start_idx
36783695
n_tokens_out = ctypes.c_size_t()
36793696
tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk_ptr, ctypes.byref(n_tokens_out))
@@ -3689,14 +3706,11 @@ def __call__(
36893706
llama.eval(tokens_to_eval)
36903707
n_past = llama.n_tokens
36913708

3692-
elif chunk_type in [
3693-
self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE,
3694-
self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO
3695-
]:
3709+
elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type):
36963710
chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk_ptr)
36973711

36983712
if self.verbose:
3699-
media_str = "IMAGE" if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE else "AUDIO"
3713+
media_str = "IMAGE" if self._is_image_chunk(chunk_type) else "AUDIO"
37003714
print(f"{self.log_prefix}(__call__): Evaluating {media_str} chunk ({chunk_n_tokens} tokens) at pos {llama.n_tokens}...", file=sys.stderr)
37013715

37023716
# Stage 5: Multimodal Physical OOM Defense

0 commit comments

Comments
 (0)