mtmd: replace hardcoded magic numbers with GGUF hparams for granite speech

Author: ReinforcedKnowledge

convert_hf_to_gguf.py

@@ -12385,6 +12385,15 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GRANITE_SPEECH)
         self.gguf_writer.add_audio_num_mel_bins(a["input_dim"])
         self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+        self.gguf_writer.add_audio_chunk_size(a["context_size"])
+        self.gguf_writer.add_audio_conv_kernel_size(a["conv_kernel_size"])
+        self.gguf_writer.add_audio_max_pos_emb(a["max_pos_emb"])
+
+        p = self.global_config
+        self.gguf_writer.add_audio_projector_window_size(p["window_size"])
+        self.gguf_writer.add_audio_projector_downsample_rate(p["downsample_rate"])
+        self.gguf_writer.add_audio_projector_head_count(p["projector_config"]["num_attention_heads"])
+        self.gguf_writer.add_audio_projector_layernorm_eps(p["projector_config"]["layer_norm_eps"])
 
     def tensor_force_quant(self, name, new_name, bid, n_dims):
         if "encoder" in name or "projector" in name:

gguf-py/gguf/constants.py

@@ -338,13 +338,20 @@ class ClipAudio:
         FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length"
         PROJECTION_DIM      = "clip.audio.projection_dim"
         BLOCK_COUNT         = "clip.audio.block_count"
+        CHUNK_SIZE          = "clip.audio.chunk_size"
+        CONV_KERNEL_SIZE    = "clip.audio.conv_kernel_size"
+        MAX_POS_EMB         = "clip.audio.max_pos_emb"
 
         class Attention:
             HEAD_COUNT      = "clip.audio.attention.head_count"
             LAYERNORM_EPS   = "clip.audio.attention.layer_norm_epsilon"
 
         class Projector:
             STACK_FACTOR    = "clip.audio.projector.stack_factor"
+            WINDOW_SIZE     = "clip.audio.projector.window_size"
+            DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate"
+            HEAD_COUNT      = "clip.audio.projector.head_count"
+            LAYERNORM_EPS   = "clip.audio.projector.layer_norm_epsilon"
 
     class Diffusion:
         SHIFT_LOGITS        = "diffusion.shift_logits"

gguf-py/gguf/gguf_writer.py

@@ -1257,6 +1257,27 @@ def add_audio_num_mel_bins(self, value: int) -> None:
     def add_audio_stack_factor(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value)
 
+    def add_audio_chunk_size(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.CHUNK_SIZE, value)
+
+    def add_audio_conv_kernel_size(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.CONV_KERNEL_SIZE, value)
+
+    def add_audio_max_pos_emb(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.MAX_POS_EMB, value)
+
+    def add_audio_projector_window_size(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.Projector.WINDOW_SIZE, value)
+
+    def add_audio_projector_downsample_rate(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.Projector.DOWNSAMPLE_RATE, value)
+
+    def add_audio_projector_head_count(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value)
+
+    def add_audio_projector_layernorm_eps(self, value: float) -> None:
+        self.add_float32(Keys.ClipAudio.Projector.LAYERNORM_EPS, value)
+
     def add_xielu_alpha_p(self, values: Sequence[float]):
         self.add_array(Keys.xIELU.ALPHA_P, values)
 

tools/mtmd/clip-impl.h

@@ -63,6 +63,13 @@
 #define KEY_AUDIO_PROJ_TYPE     "clip.audio.projector_type" // for models with mixed modalities
 #define KEY_A_NUM_MEL_BINS      "clip.audio.num_mel_bins"
 #define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"
+#define KEY_A_CHUNK_SIZE            "clip.audio.chunk_size"
+#define KEY_A_CONV_KERNEL_SIZE      "clip.audio.conv_kernel_size"
+#define KEY_A_MAX_POS_EMB           "clip.audio.max_pos_emb"
+#define KEY_A_PROJ_WINDOW_SIZE      "clip.audio.projector.window_size"
+#define KEY_A_PROJ_DOWNSAMPLE_RATE  "clip.audio.projector.downsample_rate"
+#define KEY_A_PROJ_HEAD_COUNT       "clip.audio.projector.head_count"
+#define KEY_A_PROJ_LAYERNORM_EPS    "clip.audio.projector.layer_norm_epsilon"
 
 
 //

tools/mtmd/clip-model.h

@@ -92,6 +92,13 @@ struct clip_hparams {
     // audio
     int32_t n_mel_bins = 0; // whisper preprocessor
     int32_t proj_stack_factor = 0; // ultravox
+    int32_t audio_chunk_size       = 0;
+    int32_t audio_conv_kernel_size = 0;
+    int32_t audio_max_pos_emb      = 0;
+    int32_t audio_proj_window_size     = 0;
+    int32_t audio_proj_downsample_rate = 0;
+    int32_t audio_proj_head_count      = 0;
+    float   audio_proj_layernorm_eps   = 0.0f;
 
     // audio-to-mel preprocessor params
     int32_t audio_chunk_len   = -1; // in seconds

tools/mtmd/clip.cpp

@@ -1489,6 +1489,13 @@ struct clip_model_loader {
                         hparams.audio_n_fft            = 512;
                         hparams.audio_window_len       = 400;
                         hparams.audio_hop_len          = 160;
+                        get_u32(KEY_A_CHUNK_SIZE,           hparams.audio_chunk_size);
+                        get_u32(KEY_A_CONV_KERNEL_SIZE,     hparams.audio_conv_kernel_size);
+                        get_u32(KEY_A_MAX_POS_EMB,          hparams.audio_max_pos_emb);
+                        get_u32(KEY_A_PROJ_WINDOW_SIZE,     hparams.audio_proj_window_size);
+                        get_u32(KEY_A_PROJ_DOWNSAMPLE_RATE, hparams.audio_proj_downsample_rate);
+                        get_u32(KEY_A_PROJ_HEAD_COUNT,      hparams.audio_proj_head_count);
+                        get_f32(KEY_A_PROJ_LAYERNORM_EPS,   hparams.audio_proj_layernorm_eps);
                     } break;
                 case PROJECTOR_TYPE_JANUS_PRO:
                     {
@@ -3131,7 +3138,9 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
             } break;
         case PROJECTOR_TYPE_GRANITE_SPEECH:
             {
-                n_patches = ((img->nx + 14) / 15) * 3;
+                const int ws = ctx->model.hparams.audio_proj_window_size;
+                const int ds = ctx->model.hparams.audio_proj_downsample_rate;
+                n_patches = ((img->nx + ws - 1) / ws) * (ws / ds);
             } break;
         default:
             GGML_ABORT("unsupported projector type");
@@ -3666,8 +3675,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
             } break;
         case PROJECTOR_TYPE_GRANITE_SPEECH:
             {
-                const int context_size = 200;
-                const int max_pos_emb  = 512;
+                const int context_size = ctx->model.hparams.audio_chunk_size;
+                const int max_pos_emb  = ctx->model.hparams.audio_max_pos_emb;
 
                 std::vector<int32_t> dists(context_size * context_size);
                 for (int i = 0; i < context_size; i++) {

tools/mtmd/models/granite-speech.cpp

@@ -2,9 +2,9 @@
 
 ggml_cgraph * clip_graph_granite_speech::build() {
     const int n_frames     = img.nx;
-    const int context_size = 200;
+    const int context_size = hparams.audio_chunk_size;
     const int ctc_layer    = n_layer / 2;
-    const int conv_kernel  = 15;
+    const int conv_kernel  = hparams.audio_conv_kernel_size;
     const int conv_pad     = conv_kernel / 2;
 
     const int num_blocks  = (n_frames + context_size - 1) / context_size;
@@ -185,12 +185,12 @@ ggml_cgraph * clip_graph_granite_speech::build() {
 
     // QFormer projector
     {
-        const int window_size     = 15;
-        const int num_queries     = 3;
-        const int proj_n_head     = 16;
+        const int window_size     = hparams.audio_proj_window_size;
+        const int num_queries     = window_size / hparams.audio_proj_downsample_rate;
+        const int proj_n_head     = hparams.audio_proj_head_count;
         const int proj_d_head     = n_embd / proj_n_head;
         const float proj_kq_scale = 1.0f / sqrtf((float)proj_d_head);
-        const float proj_eps      = 1e-12f;
+        const float proj_eps      = hparams.audio_proj_layernorm_eps;
         const int nblocks_proj    = (n_frames + window_size - 1) / window_size;
         const int padded_proj     = nblocks_proj * window_size;