server: cleanup (remove slot.batch_spec, rename)

srogmann · srogmann · commit a1584ac80f6f · 2026-01-24T15:55:02.000+01:00
diff --git a/common/arg.cpp b/common/arg.cpp
@@ -629,7 +629,8 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
     {
         bool has_draft =!params.speculative.model.path.empty();
         bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
-        bool has_lookup_caches = !params.lookup_cache_static.empty() && !params.lookup_cache_dynamic.empty();
+        bool has_lookup_caches = !params.speculative.lookup_cache_static.empty()
+            && !params.speculative.lookup_cache_dynamic.empty();
         bool has_simple = (params.speculative.draftless_type == COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE);
         bool found_config_draft = false;
         bool found_config_eagle3 = false;
@@ -1253,14 +1254,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"-lcs", "--lookup-cache-static"}, "FNAME",
         "path to static lookup cache to use for lookup decoding (not updated by generation)",
         [](common_params & params, const std::string & value) {
-            params.lookup_cache_static = value;
+            params.speculative.lookup_cache_static = value;
         }
     ).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"-lcd", "--lookup-cache-dynamic"}, "FNAME",
         "path to dynamic lookup cache to use for lookup decoding (updated by generation)",
         [](common_params & params, const std::string & value) {
-            params.lookup_cache_dynamic = value;
+            params.speculative.lookup_cache_dynamic = value;
         }
     ).set_examples({LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
diff --git a/common/common.h b/common/common.h
@@ -281,11 +281,16 @@ struct common_params_speculative {
 
     struct common_params_model model;
 
+    // draftless:
+
     common_speculative_type draftless_type = COMMON_SPECULATIVE_TYPE_NONE; // type of speculative decoding without a draft model
-    uint16_t spec_ngram_size_n = 12;
-    uint16_t spec_ngram_size_m = 48;
+    uint16_t spec_ngram_size_n = 12; // ngram size for lookup
+    uint16_t spec_ngram_size_m = 48; // mgram size for speculative tokens
 
     std::vector<common_speculative_config> configs = {}; // list of speculative configs to try
+
+    std::string lookup_cache_static  = ""; // path of static ngram cache file for lookup decoding           // NOLINT
+    std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding          // NOLINT
 };
 
 struct common_params_vocoder {
@@ -403,8 +408,6 @@ struct common_params {
     std::string path_prompt_cache    = ""; // path to file for saving/loading prompt eval state             // NOLINT
     std::string input_prefix         = ""; // string to prefix user inputs with                             // NOLINT
     std::string input_suffix         = ""; // string to suffix user inputs with                             // NOLINT
-    std::string lookup_cache_static  = ""; // path of static ngram cache file for lookup decoding           // NOLINT
-    std::string lookup_cache_dynamic = ""; // path of dynamic ngram cache file for lookup decoding          // NOLINT
     std::string logits_file          = ""; // file for saving *all* logits                                  // NOLINT
 
     // llama-debug specific options
diff --git a/common/ngram-map.cpp b/common/ngram-map.cpp
@@ -237,7 +237,7 @@ void common_ngram_map_draft(common_ngram_map & map,
     map.last_draft_value_idx     = slot_max; // value used for draft generation.
 }
 
-void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) {
+void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted) {
     if (!map.last_draft_created) {
         return;
     }
diff --git a/common/ngram-map.h b/common/ngram-map.h
@@ -62,5 +62,5 @@ void common_ngram_map_draft(
     const llama_tokens & inp, llama_token sampled,
     llama_tokens & draft);
 
-// Update the statistics of a value after a draft was accepted.
-void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted);
+// Update the statistics of a value after a draft was processed.
+void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted);
diff --git a/common/speculative.cpp b/common/speculative.cpp
@@ -251,12 +251,12 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
 
 
 struct common_speculative * common_speculative_init(
-        struct common_params & params,
+        struct common_params_speculative & params,
         struct llama_context * ctx_tgt,
         struct llama_context * ctx_dft
     ) {
     std::vector<std::unique_ptr<common_speculative_state>> implementations = {};
-    for (const common_speculative_config & config : params.speculative.configs) {
+    for (const common_speculative_config & config : params.configs) {
         LOG_INF("common_speculative_init: adding implementation %s\n", common_speculative_type_to_str(config.type).c_str());
         switch (config.type) {
             case COMMON_SPECULATIVE_TYPE_NONE:
@@ -271,7 +271,7 @@ struct common_speculative * common_speculative_init(
             }
             case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: {
                 common_ngram_map ngram_map = get_common_ngram_map(config,
-                        params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m);
+                        params.spec_ngram_size_n, params.spec_ngram_size_m);
                 uint16_t ngram_size_key   = ngram_map.size_key;
                 uint16_t mgram_size_value = ngram_map.size_value;
                 uint16_t check_rate = ngram_map.check_rate;
@@ -287,14 +287,14 @@ struct common_speculative * common_speculative_init(
             case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: {
                 implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k>(
                     (config.type), get_common_ngram_map(config,
-                        params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)
+                        params.spec_ngram_size_n, params.spec_ngram_size_m)
                 ));
                 break;
             }
             case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: {
                 implementations.push_back(std::make_unique<common_speculative_state_ngram_map_k4v>(
                             (config.type), get_common_ngram_map(config,
-                                params.speculative.spec_ngram_size_n, params.speculative.spec_ngram_size_m)));
+                                params.spec_ngram_size_n, params.spec_ngram_size_m)));
                 break;
             }
             case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE: {
@@ -746,7 +746,7 @@ llama_tokens common_speculative_use_draft_model(
     return result;
 }
 
-void common_speculative_send_accepted(struct common_speculative * spec, const uint16_t n_accepted) {
+void common_speculative_accept(struct common_speculative * spec, const uint16_t n_accepted) {
     common_speculative_state * impl = spec->curr_impl;
     if (impl != nullptr) {
         if (n_accepted > 0) {
@@ -756,7 +756,7 @@ void common_speculative_send_accepted(struct common_speculative * spec, const ui
         if (impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K ||
             impl->type == COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V) {
             auto state = static_cast<struct common_speculative_state_ngram_map_k *>(impl);
-            common_ngram_map_send_accepted(state->map, n_accepted);
+            common_ngram_map_accept(state->map, n_accepted);
         }
     }
 }
diff --git a/common/speculative.h b/common/speculative.h
@@ -22,7 +22,7 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
 std::string common_speculative_type_to_str(enum common_speculative_type type);
 
 struct common_speculative * common_speculative_init(
-        struct common_params & params,
+        struct common_params_speculative & params,
         struct llama_context * ctx_tgt,
         struct llama_context * ctx_dft
 );
@@ -45,7 +45,7 @@ llama_tokens common_speculative_gen_draft(
                              llama_token   id_last);
 
 // informs the speculative decoder that n_accepted tokens were accepted by the target model
-void common_speculative_send_accepted(
+void common_speculative_accept(
         struct common_speculative * spec,
         const uint16_t n_accepted);
 
diff --git a/examples/lookup/lookup-create.cpp b/examples/lookup/lookup-create.cpp
@@ -32,9 +32,9 @@ int main(int argc, char ** argv){
 
     common_ngram_cache ngram_cache;
     common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true);
-    fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.lookup_cache_static.c_str());
+    fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.lookup_cache_static.c_str());
 
-    common_ngram_cache_save(ngram_cache, params.lookup_cache_static);
+    common_ngram_cache_save(ngram_cache, params.speculative.lookup_cache_static);
 
     return 0;
 }
diff --git a/examples/lookup/lookup-stats.cpp b/examples/lookup/lookup-stats.cpp
@@ -46,18 +46,18 @@ int main(int argc, char ** argv){
     {
         const int64_t t_start_draft_us = ggml_time_us();
 
-        if (!params.lookup_cache_static.empty()) {
+        if (!params.speculative.lookup_cache_static.empty()) {
             try {
-                ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static);
+                ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static);
             } catch (std::ifstream::failure const &) {
-                LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str());
+                LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str());
                 exit(1);
             }
         }
 
-        if (!params.lookup_cache_dynamic.empty()) {
+        if (!params.speculative.lookup_cache_dynamic.empty()) {
             try {
-                ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic);
+                ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic);
             } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program
         }
 
diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp
@@ -51,18 +51,18 @@ int main(int argc, char ** argv){
         const int64_t t_start_draft_us = ggml_time_us();
         common_ngram_cache_update(ngram_cache_context, LLAMA_NGRAM_MIN, LLAMA_NGRAM_MAX, inp, inp.size(), false);
 
-        if (!params.lookup_cache_static.empty()) {
+        if (!params.speculative.lookup_cache_static.empty()) {
             try {
-                ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static);
+                ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static);
             } catch (std::ifstream::failure const &) {
-                LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str());
+                LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str());
                 exit(1);
             }
         }
 
-        if (!params.lookup_cache_dynamic.empty()) {
+        if (!params.speculative.lookup_cache_dynamic.empty()) {
             try {
-                ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic);
+                ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic);
             } catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program
         }
 
@@ -210,7 +210,7 @@ int main(int argc, char ** argv){
 
     // Update dynamic ngram cache with context ngram cache and save it to disk:
     common_ngram_cache_merge(ngram_cache_dynamic, ngram_cache_context);
-    common_ngram_cache_save(ngram_cache_dynamic, params.lookup_cache_dynamic);
+    common_ngram_cache_save(ngram_cache_dynamic, params.speculative.lookup_cache_dynamic);
 
     LOG("\n\n");
 
diff --git a/examples/speculative-simple/speculative-simple.cpp b/examples/speculative-simple/speculative-simple.cpp
@@ -132,7 +132,7 @@ int main(int argc, char ** argv) {
     params_spec.n_reuse = llama_n_ctx(ctx_dft) - n_draft;
     params_spec.p_min   = p_min;
 
-    struct common_speculative * spec = common_speculative_init(params, ctx_tgt, ctx_dft);
+    struct common_speculative * spec = common_speculative_init(params.speculative, ctx_tgt, ctx_dft);
     for (auto &pair : params.speculative.replacements) {
         common_speculative_add_replacement_tgt_dft(spec, pair.first.c_str(), pair.second.c_str());
     }
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
@@ -48,8 +48,6 @@ enum server_state {
 struct server_slot {
     int id;
 
-    llama_batch batch_spec = {};
-
     // TODO: change to unique_ptrs for consistency:
     llama_context * ctx = nullptr;
     llama_context * ctx_dft = nullptr;
@@ -604,8 +602,6 @@ struct server_context_impl {
 
             common_speculative_free(slot.spec);
             slot.spec = nullptr;
-
-            llama_batch_free(slot.batch_spec);
         }
 
         llama_batch_free(batch);
@@ -766,16 +762,14 @@ struct server_context_impl {
             slot.prompt.tokens.has_mtmd = mctx != nullptr;
 
             if (model_dft) {
-                slot.batch_spec = llama_batch_init(params_base.speculative.n_max + 1, 0, 1);
-
                 // TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
                 slot.ctx_dft = llama_init_from_model(model_dft, cparams_dft);
                 if (slot.ctx_dft == nullptr) {
                     SRV_ERR("%s", "failed to create draft context\n");
                     return false;
                 }
 
-                slot.spec = common_speculative_init(params_base, slot.ctx, slot.ctx_dft);
+                slot.spec = common_speculative_init(params_base.speculative, slot.ctx, slot.ctx_dft);
                 if (slot.spec == nullptr) {
                     SRV_ERR("%s", "failed to create speculator\n");
                     return false;
@@ -784,7 +778,7 @@ struct server_context_impl {
                     common_speculative_add_replacement_tgt_dft(slot.spec, pair.first.c_str(), pair.second.c_str());
                 }
             } else if (params_base.speculative.configs.size() > 0) {
-                slot.spec = common_speculative_init(params_base, nullptr, nullptr);
+                slot.spec = common_speculative_init(params_base.speculative, nullptr, nullptr);
             }
 
             SLT_INF(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
@@ -1185,14 +1179,6 @@ struct server_context_impl {
 
         slot.task = std::make_unique<const server_task>(std::move(task));
 
-        // initialize draft batch
-        // TODO: rework speculative decoding [TAG_SERVER_SPEC_REWORK]
-        if (slot.can_speculate()) {
-            llama_batch_free(slot.batch_spec);
-
-            slot.batch_spec = llama_batch_init(task.params.speculative.n_max + 1, 0, 1);
-        }
-
         slot.state = slot.task->is_child()
             ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
             : SLOT_STATE_STARTED;
@@ -2817,8 +2803,8 @@ struct server_context_impl {
                 // update how many tokens out of those tested were accepted
                 slot.n_draft_accepted += ids.size() - 1;
 
-                // inform the speculative decoding about the accepted tokens
-                common_speculative_send_accepted(slot.spec, ids.size() - 1);
+                // inform the speculative decoding about the number of accepted tokens
+                common_speculative_accept(slot.spec, ids.size() - 1);
 
                 // rollback to the state before sampling the draft tokens
                 slot.prompt.tokens.keep_first(slot.prompt.n_tokens() - n_draft);

Original file line number	Diff line number	Diff line change
`@@ -237,7 +237,7 @@ void common_ngram_map_draft(common_ngram_map & map,`
`237`	`237`	`map.last_draft_value_idx = slot_max; // value used for draft generation.`
`238`	`238`	`}`
`239`	`239`
`240`		`-void common_ngram_map_send_accepted(common_ngram_map & map, uint16_t n_accepted) {`
	`240`	`+void common_ngram_map_accept(common_ngram_map & map, uint16_t n_accepted) {`
`241`	`241`	`if (!map.last_draft_created) {`
`242`	`242`	`return;`
`243`	`243`	`}`
Original file line number	Diff line number	Diff line change
`@@ -32,9 +32,9 @@ int main(int argc, char ** argv){`
`32`	`32`
`33`	`33`	`common_ngram_cache ngram_cache;`
`34`	`34`	`common_ngram_cache_update(ngram_cache, LLAMA_NGRAM_STATIC, LLAMA_NGRAM_STATIC, inp, inp.size(), true);`
`35`		`- fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.lookup_cache_static.c_str());`
	`35`	`+ fprintf(stderr, "%s: hashing done, writing file to %s\n", __func__, params.speculative.lookup_cache_static.c_str());`
`36`	`36`
`37`		`- common_ngram_cache_save(ngram_cache, params.lookup_cache_static);`
	`37`	`+ common_ngram_cache_save(ngram_cache, params.speculative.lookup_cache_static);`
`38`	`38`
`39`	`39`	`return 0;`
`40`	`40`	`}`
Original file line number	Diff line number	Diff line change
`@@ -46,18 +46,18 @@ int main(int argc, char ** argv){`
`46`	`46`	`{`
`47`	`47`	`const int64_t t_start_draft_us = ggml_time_us();`
`48`	`48`
`49`		`- if (!params.lookup_cache_static.empty()) {`
	`49`	`+ if (!params.speculative.lookup_cache_static.empty()) {`
`50`	`50`	`try {`
`51`		`- ngram_cache_static = common_ngram_cache_load(params.lookup_cache_static);`
	`51`	`+ ngram_cache_static = common_ngram_cache_load(params.speculative.lookup_cache_static);`
`52`	`52`	`} catch (std::ifstream::failure const &) {`
`53`		`- LOG_ERR("failed to open static lookup cache: %s", params.lookup_cache_static.c_str());`
	`53`	`+ LOG_ERR("failed to open static lookup cache: %s", params.speculative.lookup_cache_static.c_str());`
`54`	`54`	`exit(1);`
`55`	`55`	`}`
`56`	`56`	`}`
`57`	`57`
`58`		`- if (!params.lookup_cache_dynamic.empty()) {`
	`58`	`+ if (!params.speculative.lookup_cache_dynamic.empty()) {`
`59`	`59`	`try {`
`60`		`- ngram_cache_dynamic = common_ngram_cache_load(params.lookup_cache_dynamic);`
	`60`	`+ ngram_cache_dynamic = common_ngram_cache_load(params.speculative.lookup_cache_dynamic);`
`61`	`61`	`} catch (std::ifstream::failure const &) {} // if the file does not exist it will simply be created at the end of the program`
`62`	`62`	`}`
`63`	`63`
Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ int main(int argc, char ** argv) {`
`132`	`132`	`params_spec.n_reuse = llama_n_ctx(ctx_dft) - n_draft;`
`133`	`133`	`params_spec.p_min = p_min;`
`134`	`134`
`135`		`- struct common_speculative * spec = common_speculative_init(params, ctx_tgt, ctx_dft);`
	`135`	`+ struct common_speculative * spec = common_speculative_init(params.speculative, ctx_tgt, ctx_dft);`
`136`	`136`	`for (auto &pair : params.speculative.replacements) {`
`137`	`137`	`common_speculative_add_replacement_tgt_dft(spec, pair.first.c_str(), pair.second.c_str());`
`138`	`138`	`}`