Skip to content

Commit 569d706

Browse files
TheTomclaude
authored andcommitted
fork: reconcile MTP lineage with TurboQuant+ KV cache
Integration glue so the upstream MTP lineage (ggml-org#23198..ggml-org#23398) builds on this fork without disturbing TurboQuant+ or the custom kernels: - llama_kv_cache ctor: thread the new `hparams` param and `layer_share_cb` through all call sites (iswa, memory-hybrid, dsa, model.cpp); keep the fork's turbo auto-asymmetric K upgrade, n_layer_kv() sizing (+3 rotation tensors), and per-side LLAMA_ATTN_ROT_* policy (default OFF) — now nested under the new `if (other) { share } else { ... }` KV-sharing branch. - hparams: carry n_layer_all/n_layer_nextn + n_layer()/n_layer_kv() from the refactor while keeping the fork's n_layer_kv_from_start; restore the swa_layers->is_swa_impl / recurrent_layer_arr->is_recr_impl / nextn_predict_layers->n_layer_nextn renames across fork models. - add n_outputs_max to cparams / common_params / llama_context_params and wire it through; restore deepstack_mapping_arr. - server: keep the ggml-org#23398 ctx_other (MTP draft KV-sharing) wiring; drop the ggml-org#23988 --fit VRAM pre-estimation block (depends on upstream helpers not on this fork; MTP does not need it). - drop upstream-only models pulled in by the refactor (deepseek32, mellum, talkie); keep non-MTP fork models on their own source + mechanical refactor. Builds clean on Metal; turbo quant unit test passes (turbo2/3/4 round-trip). Kernels (ggml-cuda / ggml-metal) untouched.
1 parent 535ac21 commit 569d706

8 files changed

Lines changed: 22 additions & 289 deletions

File tree

common/common.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1565,6 +1565,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
15651565

15661566
cparams.n_ctx = params.n_ctx;
15671567
cparams.n_seq_max = params.n_parallel;
1568+
cparams.n_outputs_max = params.n_outputs_max;
15681569
cparams.n_rs_seq = params.speculative.need_n_rs_seq();
15691570
cparams.n_outputs_max = std::max(params.n_outputs_max, 0);
15701571
cparams.n_batch = params.n_batch;

common/common.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ struct common_params {
431431
int32_t n_keep = 0; // number of tokens to keep from initial prompt
432432
int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
433433
int32_t n_parallel = 1; // number of parallel sequences to decode
434+
int32_t n_outputs_max = 0; // max outputs supported by the context (0 = derive)
434435
int32_t n_sequences = 1; // number of sequences to decode
435436
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
436437
int32_t grp_attn_n = 1; // group-attention factor

include/llama.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ extern "C" {
340340
uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode
341341
uint32_t n_ubatch; // physical maximum batch size
342342
uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
343+
uint32_t n_outputs_max; // max outputs supported by the context (0 = derive from n_seq_max)
343344
uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
344345
uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch)
345346
int32_t n_threads; // number of threads to use for generation

src/llama-context.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ llama_context::llama_context(
4747
const auto & hparams = model.hparams;
4848

4949
cparams.n_seq_max = std::max(1u, params.n_seq_max);
50+
cparams.n_outputs_max = params.n_outputs_max;
5051
if (cparams.n_seq_max > LLAMA_MAX_SEQ) {
5152
throw std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ));
5253
}
@@ -3446,6 +3447,7 @@ llama_context_params llama_context_default_params() {
34463447
/*.n_batch =*/ 2048,
34473448
/*.n_ubatch =*/ 512,
34483449
/*.n_seq_max =*/ 1,
3450+
/*.n_outputs_max =*/ 0,
34493451
/*.n_rs_seq =*/ 0,
34503452
/*.n_outputs_max =*/ 0,
34513453
/*.n_threads =*/ GGML_DEFAULT_N_THREADS, // TODO: better default

src/llama-hparams.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,16 @@ uint32_t llama_hparams::n_layer() const {
282282
return n_layer_all - n_layer_nextn;
283283
}
284284

285+
uint32_t llama_hparams::n_layer_kv() const {
286+
uint32_t res = 0;
287+
for (uint32_t il = 0; il < n_layer(); ++il) {
288+
if (has_kv(il)) {
289+
res++;
290+
}
291+
}
292+
return res;
293+
}
294+
285295
bool llama_hparams::use_mrope() const {
286296
return rope_sections[0] > 0 && rope_sections[1] > 0;
287297
}

src/llama-hparams.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,9 @@ struct llama_hparams {
346346
// number of effective layers (excludes nextn layers)
347347
uint32_t n_layer() const;
348348

349+
// number of layers that carry a KV cache (respects n_layer_kv_from_start)
350+
uint32_t n_layer_kv() const;
351+
349352
// note that this function uses different SWA parameters from those in the hparams
350353
// note: inlined on purpose for performance reasons
351354
// TODO: think of a better place for this function

src/llama-kv-cache.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,15 +321,15 @@ llama_kv_cache::llama_kv_cache(
321321
return mode;
322322
}
323323
// Auto-enable Boundary V (mode 7) when V is turbo2
324-
if (type_v == GGML_TYPE_TURBO2_0 && hparams.n_layer >= 8) {
324+
if (type_v == GGML_TYPE_TURBO2_0 && hparams.n_layer() >= 8) {
325325
LLAMA_LOG_INFO("llama_kv_cache: Boundary V auto-enabled for turbo2-V (opt-out: TURBO_LAYER_ADAPTIVE=0)\n");
326326
return 7;
327327
}
328328
return 0;
329329
}();
330330
const bool is_turbo = (type_k == GGML_TYPE_TURBO3_0 || type_k == GGML_TYPE_TURBO4_0 || type_k == GGML_TYPE_TURBO2_0);
331331
const bool v_is_turbo = (type_v == GGML_TYPE_TURBO3_0 || type_v == GGML_TYPE_TURBO4_0 || type_v == GGML_TYPE_TURBO2_0);
332-
const uint32_t n_layer = hparams.n_layer;
332+
const uint32_t n_layer = hparams.n_layer();
333333
if (adaptive_mode == 1 && is_turbo && n_layer >= 8) {
334334
if (il < 4 || il >= n_layer - 4) {
335335
layer_type_k = GGML_TYPE_Q8_0;

0 commit comments

Comments
 (0)