Skip to content

Commit 84bd03a

Browse files
ggerganovggerganov
committed
talk-llama : sync llama.cpp
1 parent 4df9a57 commit 84bd03a

151 files changed

Lines changed: 3434 additions & 866 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎examples/talk-llama/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ if (WHISPER_SDL2)
2020
llama-io.cpp
2121
llama-kv-cache.cpp
2222
llama-kv-cache-iswa.cpp
23+
llama-kv-cache-dsa.cpp
2324
llama-memory-recurrent.cpp
2425
llama-memory-hybrid.cpp
2526
llama-memory-hybrid-iswa.cpp

‎examples/talk-llama/llama-adapter.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ bool llama_adapter_cvec::init(const llama_model & model) {
4141
auto it = ctx_map.find(buft);
4242
if (it == ctx_map.end()) {
4343
ggml_init_params params = {
44-
/*.mem_size =*/ hparams.n_layer*ggml_tensor_overhead(),
44+
/*.mem_size =*/ hparams.n_layer()*ggml_tensor_overhead(),
4545
/*.mem_buffer =*/ NULL,
4646
/*.no_alloc =*/ true,
4747
};
@@ -61,9 +61,9 @@ bool llama_adapter_cvec::init(const llama_model & model) {
6161
};
6262

6363
// make tensors
64-
tensors.reserve(hparams.n_layer);
64+
tensors.reserve(hparams.n_layer());
6565
tensors.push_back(nullptr); // there's never a tensor for layer 0
66-
for (size_t il = 1; il < hparams.n_layer; il++) {
66+
for (size_t il = 1; il < hparams.n_layer(); il++) {
6767
ggml_backend_buffer_type_t buft = model.select_buft(il);
6868
ggml_context * ctx = ctx_for_buft(buft);
6969
if (!ctx) {
@@ -121,7 +121,7 @@ bool llama_adapter_cvec::apply(
121121
layer_start = il_start;
122122
layer_end = il_end;
123123

124-
for (size_t il = 1; il < hparams.n_layer; il++) {
124+
for (size_t il = 1; il < hparams.n_layer(); il++) {
125125
assert(tensors[il] != nullptr);
126126

127127
const size_t off = n_embd * (il - 1); // buffer doesn't have data for layer 0, since it's never present

‎examples/talk-llama/llama-arch.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
5757
{ LLM_ARCH_GEMMA3, "gemma3" },
5858
{ LLM_ARCH_GEMMA3N, "gemma3n" },
5959
{ LLM_ARCH_GEMMA4, "gemma4" },
60+
{ LLM_ARCH_GEMMA4_ASSISTANT, "gemma4-assistant" },
6061
{ LLM_ARCH_GEMMA_EMBEDDING, "gemma-embedding" },
6162
{ LLM_ARCH_STARCODER2, "starcoder2" },
6263
{ LLM_ARCH_MAMBA, "mamba" },
@@ -75,6 +76,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
7576
{ LLM_ARCH_DEEPSEEK, "deepseek" },
7677
{ LLM_ARCH_DEEPSEEK2, "deepseek2" },
7778
{ LLM_ARCH_DEEPSEEK2OCR, "deepseek2-ocr" },
79+
{ LLM_ARCH_DEEPSEEK32, "deepseek32" },
7880
{ LLM_ARCH_CHATGLM, "chatglm" },
7981
{ LLM_ARCH_GLM4, "glm4" },
8082
{ LLM_ARCH_GLM4_MOE, "glm4moe" },
@@ -134,6 +136,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
134136
{ LLM_ARCH_MAINCODER, "maincoder" },
135137
{ LLM_ARCH_KIMI_LINEAR, "kimi-linear" },
136138
{ LLM_ARCH_TALKIE, "talkie" },
139+
{ LLM_ARCH_MELLUM, "mellum" },
137140
{ LLM_ARCH_UNKNOWN, "(unknown)" },
138141
};
139142

@@ -194,6 +197,8 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
194197
{ LLM_KV_MOE_LATENT_SIZE, "%s.moe_latent_size" },
195198
{ LLM_KV_NEXTN_PREDICT_LAYERS, "%s.nextn_predict_layers" },
196199
{ LLM_KV_NUM_DEEPSTACK_LAYERS, "%s.n_deepstack_layers" },
200+
{ LLM_KV_DEEPSTACK_MAPPING, "%s.deepstack_mapping" },
201+
{ LLM_KV_HIDDEN_ACT, "%s.hidden_activation" },
197202
{ LLM_KV_POOLING_TYPE, "%s.pooling_type" },
198203
{ LLM_KV_LOGIT_SCALE, "%s.logit_scale" },
199204
{ LLM_KV_DECODER_START_TOKEN_ID, "%s.decoder_start_token_id" },
@@ -244,6 +249,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
244249
{ LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, "%s.attention.indexer.key_length" },
245250
{ LLM_KV_ATTENTION_INDEXER_TOP_K, "%s.attention.indexer.top_k" },
246251
{ LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" },
252+
{ LLM_KV_ATTENTION_RECURRENT_LAYERS, "%s.attention.recurrent_layers" },
247253

248254
{ LLM_KV_ROPE_DIMENSION_COUNT, "%s.rope.dimension_count" },
249255
{ LLM_KV_ROPE_DIMENSION_COUNT_SWA, "%s.rope.dimension_count_swa" },
@@ -318,12 +324,14 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
318324
{ LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" },
319325
{ LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" },
320326
{ LLM_KV_TOKENIZER_CHAT_TEMPLATE, "tokenizer.chat_template" },
327+
{ LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE, "tokenizer.ggml.normalizer.lowercase" },
321328
{ LLM_KV_TOKENIZER_FIM_PRE_ID, "tokenizer.ggml.fim_pre_token_id" },
322329
{ LLM_KV_TOKENIZER_FIM_SUF_ID, "tokenizer.ggml.fim_suf_token_id" },
323330
{ LLM_KV_TOKENIZER_FIM_MID_ID, "tokenizer.ggml.fim_mid_token_id" },
324331
{ LLM_KV_TOKENIZER_FIM_PAD_ID, "tokenizer.ggml.fim_pad_token_id" },
325332
{ LLM_KV_TOKENIZER_FIM_REP_ID, "tokenizer.ggml.fim_rep_token_id" },
326333
{ LLM_KV_TOKENIZER_FIM_SEP_ID, "tokenizer.ggml.fim_sep_token_id" },
334+
{ LLM_KV_TOKENIZER_SUPPRESS_TOKENS, "tokenizer.ggml.suppress_tokens" },
327335

328336
{ LLM_KV_ADAPTER_TYPE, "adapter.type" },
329337
{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
@@ -446,6 +454,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
446454
{ LLM_TENSOR_FFN_NORM_EXPS, "blk.%d.ffn_norm_exps" },
447455
{ LLM_TENSOR_ATTN_K_B, "blk.%d.attn_k_b" },
448456
{ LLM_TENSOR_ATTN_V_B, "blk.%d.attn_v_b" },
457+
{ LLM_TENSOR_NEXTN_PROJ_PRE, "nextn.pre_projection" },
458+
{ LLM_TENSOR_NEXTN_PROJ_POST, "nextn.post_projection" },
449459
{ LLM_TENSOR_NEXTN_EH_PROJ, "blk.%d.nextn.eh_proj" },
450460
{ LLM_TENSOR_NEXTN_EMBED_TOKENS, "blk.%d.nextn.embed_tokens" },
451461
{ LLM_TENSOR_NEXTN_ENORM, "blk.%d.nextn.enorm" },
@@ -758,6 +768,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
758768
{LLM_TENSOR_INDEXER_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
759769
{LLM_TENSOR_INDEXER_ATTN_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
760770
{LLM_TENSOR_INDEXER_ATTN_Q_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
771+
{LLM_TENSOR_NEXTN_PROJ_PRE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
772+
{LLM_TENSOR_NEXTN_PROJ_POST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
761773
// NextN/MTP tensors are stored per-block (blk.%d.nextn.*) even though only the
762774
// last nextn_predict_layers blocks carry them. Classify as LAYER_REPEATING so
763775
// the model loader doesn't fault on the block index.
@@ -904,6 +916,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
904916
case LLM_ARCH_OLMO2:
905917
case LLM_ARCH_OLMOE:
906918
case LLM_ARCH_DEEPSEEK2:
919+
case LLM_ARCH_DEEPSEEK32:
907920
case LLM_ARCH_GLM_DSA:
908921
case LLM_ARCH_BITNET:
909922
case LLM_ARCH_T5:

‎examples/talk-llama/llama-arch.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ enum llm_arch {
6161
LLM_ARCH_GEMMA3,
6262
LLM_ARCH_GEMMA3N,
6363
LLM_ARCH_GEMMA4,
64+
LLM_ARCH_GEMMA4_ASSISTANT,
6465
LLM_ARCH_GEMMA_EMBEDDING,
6566
LLM_ARCH_STARCODER2,
6667
LLM_ARCH_MAMBA,
@@ -79,6 +80,7 @@ enum llm_arch {
7980
LLM_ARCH_DEEPSEEK,
8081
LLM_ARCH_DEEPSEEK2,
8182
LLM_ARCH_DEEPSEEK2OCR,
83+
LLM_ARCH_DEEPSEEK32,
8284
LLM_ARCH_CHATGLM,
8385
LLM_ARCH_GLM4,
8486
LLM_ARCH_GLM4_MOE,
@@ -138,6 +140,7 @@ enum llm_arch {
138140
LLM_ARCH_MAINCODER,
139141
LLM_ARCH_KIMI_LINEAR,
140142
LLM_ARCH_TALKIE,
143+
LLM_ARCH_MELLUM,
141144
LLM_ARCH_UNKNOWN,
142145
};
143146

@@ -198,6 +201,8 @@ enum llm_kv {
198201
LLM_KV_MOE_LATENT_SIZE,
199202
LLM_KV_NEXTN_PREDICT_LAYERS,
200203
LLM_KV_NUM_DEEPSTACK_LAYERS,
204+
LLM_KV_DEEPSTACK_MAPPING,
205+
LLM_KV_HIDDEN_ACT,
201206
LLM_KV_POOLING_TYPE,
202207
LLM_KV_LOGIT_SCALE,
203208
LLM_KV_DECODER_START_TOKEN_ID,
@@ -248,6 +253,7 @@ enum llm_kv {
248253
LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
249254
LLM_KV_ATTENTION_INDEXER_TOP_K,
250255
LLM_KV_ATTENTION_SHARED_KV_LAYERS,
256+
LLM_KV_ATTENTION_RECURRENT_LAYERS,
251257

252258
LLM_KV_ROPE_DIMENSION_COUNT,
253259
LLM_KV_ROPE_DIMENSION_COUNT_SWA,
@@ -307,12 +313,14 @@ enum llm_kv {
307313
LLM_KV_TOKENIZER_HF_JSON,
308314
LLM_KV_TOKENIZER_RWKV,
309315
LLM_KV_TOKENIZER_CHAT_TEMPLATE,
316+
LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
310317
LLM_KV_TOKENIZER_FIM_PRE_ID,
311318
LLM_KV_TOKENIZER_FIM_SUF_ID,
312319
LLM_KV_TOKENIZER_FIM_MID_ID,
313320
LLM_KV_TOKENIZER_FIM_PAD_ID,
314321
LLM_KV_TOKENIZER_FIM_REP_ID,
315322
LLM_KV_TOKENIZER_FIM_SEP_ID,
323+
LLM_KV_TOKENIZER_SUPPRESS_TOKENS,
316324

317325
LLM_KV_ADAPTER_TYPE,
318326
LLM_KV_ADAPTER_LORA_ALPHA,
@@ -550,6 +558,8 @@ enum llm_tensor {
550558
LLM_TENSOR_INDEXER_PROJ,
551559
LLM_TENSOR_INDEXER_ATTN_K,
552560
LLM_TENSOR_INDEXER_ATTN_Q_B,
561+
LLM_TENSOR_NEXTN_PROJ_PRE,
562+
LLM_TENSOR_NEXTN_PROJ_POST,
553563
LLM_TENSOR_NEXTN_EH_PROJ,
554564
LLM_TENSOR_NEXTN_EMBED_TOKENS,
555565
LLM_TENSOR_NEXTN_ENORM,

0 commit comments

Comments
 (0)