Fix MTP text requests with mmproj loaded

Author: Anbeeld

tests/test-mtmd-plumbing.cpp

@@ -87,6 +87,10 @@ int main(int argc, char ** argv) {
     ok &= expect(server_context.find("params_base.n_batch = std::max(params_base.n_batch, mmproj_decode_req.min_decoder_batch_tokens)") != std::string::npos &&
                  server_context.find("params_base.n_ubatch = std::max(params_base.n_ubatch, mmproj_decode_req.min_decoder_batch_tokens)") != std::string::npos,
         "server must raise batch and ubatch before context creation for full non-causal image chunks");
+    ok &= expect(server_context.find("if (mctx && params_base.speculative.type() != COMMON_SPECULATIVE_TYPE_DFLASH) {\n                    GGML_ABORT(\"not supported by multimodal\");") == std::string::npos,
+        "server must not abort non-DFlash speculative text requests just because mmproj is loaded");
+    ok &= expect(server_context.find("slot.prompt.tokens.has_media()") != std::string::npos,
+        "server must gate non-DFlash speculative multimodal fallback on actual media in the active prompt");
 
     ok &= expect(mtmd_helper.find("const int32_t n_ubatch = llama_n_ubatch(lctx)") != std::string::npos,
         "mtmd image decode must inspect the decoder physical ubatch");

tools/server/server-common.h

@@ -203,6 +203,8 @@ struct server_tokens {
 
     bool empty() const { return tokens.empty(); }
 
+    bool has_media() const { return !map_idx_to_media.empty(); }
+
     void clear() {
         map_idx_to_media.clear();
         tokens.clear();

tools/server/server-context.cpp

@@ -4638,14 +4638,18 @@ struct server_context_impl {
             const bool dflash_sampler_blocks_speculative =
                 params_base.speculative.type() == COMMON_SPECULATIVE_TYPE_DFLASH &&
                 slot.smpl && common_sampler_reasoning_is_forcing(slot.smpl.get());
+            const bool non_dflash_media_prompt =
+                mctx &&
+                params_base.speculative.type() != COMMON_SPECULATIVE_TYPE_DFLASH &&
+                slot.prompt.tokens.has_media();
+            if (non_dflash_media_prompt) {
+                SLT_DBG(slot, "%s", "non-DFlash speculative decoding does not support media inputs; using regular decode\n");
+                n_draft_max = 0;
+            }
 
             if (n_draft_max > 0 && !dflash_sampler_blocks_speculative) {
                 const int64_t t_draft_slot_start = ggml_time_us();
 
-                if (mctx && params_base.speculative.type() != COMMON_SPECULATIVE_TYPE_DFLASH) {
-                    GGML_ABORT("not supported by multimodal");
-                }
-
                 auto params_spec = slot.task->params.speculative;
                 const llama_tokens & cached_text_tokens = slot.prompt.tokens.get_text_tokens();
                 const int original_n_max = params_spec.n_max;