Skip to content

Commit f1d976f

Browse files
committed
fix(speculative): disable backend sampling for MTP drafts
Assisted-by: Claude:opus-4.8 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 2fed6aa commit f1d976f

1 file changed

Lines changed: 6 additions & 0 deletions

File tree

common/speculative.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -952,6 +952,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
952952
ctx_dft ? "yes" : "no",
953953
common_speculative_get_devices_str(this->params.devices).c_str());
954954

955+
if (this->params.backend_sampling) {
956+
LOG_WRN("%s: backend draft sampling is disabled for MTP; verification batches can request multiple output rows per sequence\n",
957+
__func__);
958+
this->params.backend_sampling = false;
959+
}
960+
955961
const int32_t n_b = (int32_t) llama_n_batch(ctx_dft);
956962
batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd, /*n_seq_max=*/ 1);
957963
// llama_batch_init allocates only one of token/embd; MTP needs both.

0 commit comments

Comments
 (0)