We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 2fed6aa commit f1d976fCopy full SHA for f1d976f
1 file changed
common/speculative.cpp
@@ -952,6 +952,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
952
ctx_dft ? "yes" : "no",
953
common_speculative_get_devices_str(this->params.devices).c_str());
954
955
+ if (this->params.backend_sampling) {
956
+ LOG_WRN("%s: backend draft sampling is disabled for MTP; verification batches can request multiple output rows per sequence\n",
957
+ __func__);
958
+ this->params.backend_sampling = false;
959
+ }
960
+
961
const int32_t n_b = (int32_t) llama_n_batch(ctx_dft);
962
batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd, /*n_seq_max=*/ 1);
963
// llama_batch_init allocates only one of token/embd; MTP needs both.
0 commit comments