Skip to content

Commit fdb1db8

Browse files
angtangt
authored
llama : add llama_model_ftype_name() (ggml-org#25134)
* llama : add llama_model_ftype_name() Expose the model file type (quantization) name, e.g. "Q8_0" or "Q4_K - Medium", through a new public C API. The returned pointer is valid for the lifetime of the model and nullptr when the model is invalid or the file type is unknown. Signed-off-by: Adrien Gallouët <angt@huggingface.co> * Export enum Signed-off-by: Adrien Gallouët <angt@huggingface.co> * s/llama_model_ftype_name/llama_ftype_name/ Signed-off-by: Adrien Gallouët <angt@huggingface.co> * Move "(guessed)" to the front in llama_ftype_name Prepend the "(guessed)" label instead of appending it. This allows removing the non-thread-safe static std::string, making the function allocation-free. Signed-off-by: Adrien Gallouët <angt@huggingface.co> * Add LLAMA_FTYPE_PREFIX Signed-off-by: Adrien Gallouët <angt@huggingface.co> * Dont check for model Signed-off-by: Adrien Gallouët <angt@huggingface.co> --------- Signed-off-by: Adrien Gallouët <angt@huggingface.co>
1 parent 4fc4ec5 commit fdb1db8

7 files changed

Lines changed: 75 additions & 45 deletions

File tree

‎include/llama.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,9 @@ extern "C" {
159159
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
160160
};
161161

162+
// Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
163+
LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);
164+
162165
enum llama_rope_scaling_type {
163166
LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
164167
LLAMA_ROPE_SCALING_TYPE_NONE = 0,
@@ -606,6 +609,9 @@ extern "C" {
606609
// Get a string describing the model type
607610
LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
608611

612+
// Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0
613+
LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);
614+
609615
// Returns the total size of all the tensors in the model in bytes
610616
LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
611617

‎src/llama-model-loader.cpp‎

Lines changed: 47 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -27,52 +27,54 @@ const char * llama_file_version_name(llama_fver version) {
2727
return "unknown";
2828
}
2929

30-
static std::string llama_model_ftype_name(llama_ftype ftype) {
31-
if (ftype & LLAMA_FTYPE_GUESSED) {
32-
return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";
33-
}
34-
35-
switch (ftype) {
36-
case LLAMA_FTYPE_ALL_F32: return "all F32";
37-
case LLAMA_FTYPE_MOSTLY_F16: return "F16";
38-
case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
39-
case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";
40-
case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
41-
case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
42-
case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
43-
case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";
44-
case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";
45-
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE";
46-
case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4";
47-
case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";
48-
case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";
49-
case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";
50-
case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";
51-
case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";
52-
case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";
53-
case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";
54-
case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";
55-
case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";
56-
case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";
57-
case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";
58-
case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";
59-
case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";
60-
case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";
61-
case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";
62-
case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";
63-
case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";
64-
case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";
65-
case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";
66-
case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";
67-
case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";
68-
case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";
69-
case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";
70-
case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";
71-
72-
default: return "unknown, may not work";
30+
#define LLAMA_FTYPE_PREFIX "(guessed) "
31+
32+
const char * llama_ftype_name(llama_ftype ftype) {
33+
static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1;
34+
const char * name;
35+
switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) {
36+
case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break;
37+
case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break;
38+
case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break;
39+
case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;
40+
case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;
41+
case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;
42+
case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;
43+
case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break;
44+
case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break;
45+
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break;
46+
case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break;
47+
case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break;
48+
case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break;
49+
case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break;
50+
case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break;
51+
case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break;
52+
case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break;
53+
case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break;
54+
case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break;
55+
case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break;
56+
case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break;
57+
case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break;
58+
case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break;
59+
case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break;
60+
case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break;
61+
case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break;
62+
case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break;
63+
case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break;
64+
case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break;
65+
case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break;
66+
case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break;
67+
case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break;
68+
case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break;
69+
case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break;
70+
case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break;
71+
default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break;
7372
}
73+
return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len;
7474
}
7575

76+
#undef LLAMA_FTYPE_PREFIX
77+
7678
// return a list of splits for a given path
7779
// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits
7880
static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {
@@ -1693,12 +1695,12 @@ bool llama_model_loader::load_all_data(
16931695
}
16941696

16951697
std::string llama_model_loader::ftype_name() const {
1696-
return llama_model_ftype_name(ftype);
1698+
return llama_ftype_name(ftype);
16971699
}
16981700

16991701
void llama_model_loader::print_info() const {
17001702
LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver));
1701-
LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str());
1703+
LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_ftype_name(ftype));
17021704
if (n_bytes < GiB) {
17031705
LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);
17041706
} else {

‎src/llama-model.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -987,6 +987,8 @@ struct llama_model::impl {
987987

988988
std::string desc_str;
989989

990+
llama_ftype ftype = LLAMA_FTYPE_ALL_F32;
991+
990992
// model memory mapped files
991993
llama_mmaps mappings;
992994

@@ -1200,6 +1202,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
12001202

12011203
pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name();
12021204

1205+
pimpl->ftype = ml.ftype;
1206+
12031207
if (hparams.f_max_alibi_bias > 0.0f) {
12041208
hparams.use_alibi = true;
12051209
}
@@ -1646,6 +1650,10 @@ std::string llama_model::desc() const {
16461650
return pimpl->desc_str;
16471651
}
16481652

1653+
llama_ftype llama_model::ftype() const {
1654+
return pimpl->ftype;
1655+
}
1656+
16491657
size_t llama_model::size() const {
16501658
return pimpl->n_bytes;
16511659
}
@@ -2616,6 +2624,10 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size)
26162624
return snprintf(buf, buf_size, "%s", model->desc().c_str());
26172625
}
26182626

2627+
llama_ftype llama_model_ftype(const llama_model * model) {
2628+
return model->ftype();
2629+
}
2630+
26192631
uint64_t llama_model_size(const llama_model * model) {
26202632
return model->size();
26212633
}

‎src/llama-model.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -637,6 +637,8 @@ struct llama_model {
637637

638638
std::string desc() const;
639639

640+
llama_ftype ftype() const;
641+
640642
size_t size() const; // file size
641643
size_t n_tensors() const;
642644
size_t n_devices() const;

‎tools/cli/cli.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,9 @@ int llama_cli(int argc, char ** argv) {
448448
console::log("%s\n", LLAMA_ASCII_LOGO);
449449
console::log("build : %s\n", inf.build_info.c_str());
450450
console::log("model : %s\n", inf.model_name.c_str());
451+
if (!inf.model_ftype.empty()) {
452+
console::log("ftype : %s\n", inf.model_ftype.c_str());
453+
}
451454
console::log("modalities : %s\n", modalities.c_str());
452455
if (!params.system_prompt.empty()) {
453456
console::log("using custom system prompt\n");

‎tools/server/server-context.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3989,6 +3989,8 @@ server_context_meta server_context::get_meta() const {
39893989
auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : "";
39903990
auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : "";
39913991

3992+
const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));
3993+
39923994
return server_context_meta {
39933995
/* build_info */ std::string(llama_build_info()),
39943996
/* model_name */ impl->model_name,
@@ -4023,6 +4025,7 @@ server_context_meta server_context::get_meta() const {
40234025
/* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt),
40244026
/* model_n_params */ llama_model_n_params(impl->model_tgt),
40254027
/* model_size */ llama_model_size(impl->model_tgt),
4028+
/* model_ftype */ ftype_name,
40264029
};
40274030
}
40284031

@@ -5118,6 +5121,7 @@ json server_routes::get_model_info() const {
51185121
{"n_embd", meta->model_n_embd_inp},
51195122
{"n_params", meta->model_n_params},
51205123
{"size", meta->model_size},
5124+
{"ftype", meta->model_ftype},
51215125
}},
51225126
};
51235127
}

‎tools/server/server-context.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ struct server_context_meta {
5050
int32_t model_n_embd_inp;
5151
uint64_t model_n_params;
5252
uint64_t model_size;
53+
std::string model_ftype;
5354
};
5455

5556
enum server_state {

0 commit comments

Comments
 (0)