Skip to content

Commit 978ff32

Browse files
JamePengJamePeng
committed
feat(llama_ext): support optional llama-ext.h API bindings
- Add Python ctypes bindings for the experimental APIs exposed by llama-ext.h, including NextN/MTP embeddings, and model metadata extraction. - Extension symbols are loaded optionally to handle ABI changes, renamed symbols, and builds that do not export experimental APIs without breaking the main Python bindings. Signed-off-by: JamePeng <jame_peng@sina.com>
1 parent 6c163c2 commit 978ff32

1 file changed

Lines changed: 293 additions & 0 deletions

File tree

‎llama_cpp/llama_cpp.py‎

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
ggml_backend_sched_eval_callback,
1111
ggml_log_callback,
1212
ggml_opt_get_optimizer_params,
13+
ggml_cgraph
1314
)
1415

1516
from typing import (
@@ -5069,3 +5070,295 @@ def llama_opt_epoch(
50695070
callback_eval: ctypes.c_void_p, /
50705071
):
50715072
...
5073+
5074+
##############################
5075+
# // llama.cpp/src/llama-ext.h
5076+
##############################
5077+
5078+
# // this is a staging header for new llama.cpp API
5079+
# // breaking changes and C++ are allowed. everything here should be considered WIP
5080+
# // try as much as possible to not include this header in the rest of the codebase
5081+
5082+
ctypes_function_llama_ext = ctypes_function_for_shared_library(_lib)
5083+
5084+
# // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.
5085+
# LLAMA_API struct ggml_cgraph * llama_graph_reserve(
5086+
# struct llama_context * ctx,
5087+
# uint32_t n_tokens,
5088+
# uint32_t n_seqs,
5089+
# uint32_t n_outputs);
5090+
@ctypes_function_llama_ext(
5091+
[
5092+
"llama_graph_reserve",
5093+
"?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z",
5094+
"__Z19llama_graph_reserveP13llama_contextjjj",
5095+
],
5096+
[llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32],
5097+
ctypes.POINTER(ggml_cgraph),
5098+
required=False,
5099+
)
5100+
def llama_graph_reserve(
5101+
ctx: llama_context_p,
5102+
n_tokens: ctypes.c_uint32,
5103+
n_seqs: ctypes.c_uint32,
5104+
n_outputs: ctypes.c_uint32,
5105+
) -> ctypes.POINTER(ggml_cgraph): # type: ignore
5106+
"""
5107+
Reserve a new compute graph. It is valid until the next call to llama_graph_reserve.
5108+
"""
5109+
...
5110+
5111+
# // Get the default ggml_type for a given ftype.
5112+
# LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype);
5113+
@ctypes_function_llama_ext(
5114+
[
5115+
"llama_ftype_get_default_type",
5116+
"?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z",
5117+
"__Z28llama_ftype_get_default_type11llama_ftype",
5118+
],
5119+
[ctypes.c_int],
5120+
int,
5121+
required=False,
5122+
)
5123+
def llama_ftype_get_default_type(
5124+
ftype: llama_ftype
5125+
) -> int:
5126+
"""
5127+
Get the default ggml_type for a given ftype.
5128+
"""
5129+
...
5130+
5131+
# LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model);
5132+
@ctypes_function_llama_ext(
5133+
[
5134+
"llama_model_n_expert",
5135+
"?llama_model_n_expert@@YAHPEBUllama_model@@@Z",
5136+
"__Z20llama_model_n_expertPK11llama_model",
5137+
],
5138+
[llama_model_p_ctypes],
5139+
ctypes.c_int32,
5140+
required=False,
5141+
)
5142+
def llama_model_n_expert(
5143+
model: llama_model_p
5144+
) -> ctypes.c_int32:
5145+
...
5146+
5147+
# LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model);
5148+
@ctypes_function_llama_ext(
5149+
[
5150+
"llama_model_n_devices",
5151+
"?llama_model_n_devices@@YAHPEBUllama_model@@@Z",
5152+
"__Z21llama_model_n_devicesPK11llama_model",
5153+
],
5154+
[llama_model_p_ctypes],
5155+
ctypes.c_int32,
5156+
required=False,
5157+
)
5158+
def llama_model_n_devices(
5159+
model: llama_model_p
5160+
) -> ctypes.c_int32:
5161+
...
5162+
5163+
# LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i);
5164+
@ctypes_function_llama_ext(
5165+
[
5166+
"llama_model_get_device",
5167+
"?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z",
5168+
"__Z22llama_model_get_devicePK11llama_modeli",
5169+
],
5170+
[llama_model_p_ctypes, ctypes.c_int],
5171+
ctypes.c_void_p,
5172+
required=False,
5173+
)
5174+
def llama_model_get_device(
5175+
model: llama_model_p,
5176+
i: int,
5177+
) -> ctypes.c_void_p:
5178+
...
5179+
5180+
# // Set whether the context outputs nextn embeddings or not
5181+
# // If masked == true, output the embeddings only for the tokens with batch.logits != 0
5182+
# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits
5183+
# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked);
5184+
@ctypes_function_llama_ext(
5185+
[
5186+
"llama_set_embeddings_nextn",
5187+
"?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z",
5188+
"__Z26llama_set_embeddings_nextnP13llama_contextbb",
5189+
],
5190+
[llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool],
5191+
None,
5192+
required=False,
5193+
)
5194+
def llama_set_embeddings_nextn(
5195+
ctx: llama_context_p,
5196+
value: bool,
5197+
masked: bool,
5198+
):
5199+
"""
5200+
Set whether the context outputs nextn embeddings or not
5201+
If masked == true, output the embeddings only for the tokens with batch.logits != 0
5202+
If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits
5203+
"""
5204+
...
5205+
5206+
# // Select which appended NextN block the DECODER_MTP graph runs (offset past
5207+
# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to
5208+
# // chain multiple trained NextN heads. Default 0 (first head).
5209+
# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
5210+
@ctypes_function_llama_ext(
5211+
[
5212+
"llama_set_nextn_layer_offset",
5213+
"?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z",
5214+
"__Z28llama_set_nextn_layer_offsetP13llama_contexti",
5215+
],
5216+
[llama_context_p_ctypes, ctypes.c_int32],
5217+
None,
5218+
required=False,
5219+
)
5220+
def llama_set_nextn_layer_offset(
5221+
ctx: llama_context_p,
5222+
offset: ctypes.c_int32,
5223+
):
5224+
"""
5225+
Select which appended NextN block the DECODER_MTP graph runs (offset past
5226+
the trunk: il = n_layer() + offset). Used by the speculative NextN driver to
5227+
chain multiple trained NextN heads. Default 0 (first head).
5228+
"""
5229+
...
5230+
5231+
# // mirrors:
5232+
# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
5233+
# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx);
5234+
@ctypes_function_llama_ext(
5235+
[
5236+
"llama_get_embeddings_nextn",
5237+
"?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z",
5238+
"__Z26llama_get_embeddings_nextnP13llama_context",
5239+
],
5240+
[llama_context_p_ctypes],
5241+
ctypes.POINTER(ctypes.c_float),
5242+
required=False,
5243+
)
5244+
def llama_get_embeddings_nextn(
5245+
ctx: llama_context_p,
5246+
) -> ctypes.POINTER(ctypes.c_float): # type: ignore
5247+
...
5248+
5249+
# // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
5250+
# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i);
5251+
@ctypes_function_llama_ext(
5252+
[
5253+
"llama_get_embeddings_nextn_ith",
5254+
"?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z",
5255+
"__Z30llama_get_embeddings_nextn_ithP13llama_contexti",
5256+
],
5257+
[llama_context_p_ctypes, ctypes.c_int32],
5258+
ctypes.POINTER(ctypes.c_float),
5259+
required=False,
5260+
)
5261+
def llama_get_embeddings_nextn_ith(
5262+
ctx: llama_context_p,
5263+
i: ctypes.c_int32,
5264+
) -> ctypes.POINTER(ctypes.c_float): # type: ignore
5265+
...
5266+
5267+
# // Set whether the context outputs the input embeddings of a specific layer
5268+
# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value);
5269+
@ctypes_function_llama_ext(
5270+
[
5271+
"llama_set_embeddings_layer_inp",
5272+
"?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z",
5273+
"__Z30llama_set_embeddings_layer_inpP13llama_contextjb",
5274+
],
5275+
[llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool],
5276+
ctypes.POINTER(ctypes.c_float),
5277+
required=False,
5278+
)
5279+
def llama_set_embeddings_layer_inp(
5280+
ctx: llama_context_p,
5281+
lid: ctypes.c_int32,
5282+
value: bool,
5283+
) -> ctypes.POINTER(ctypes.c_float): # type: ignore
5284+
"""
5285+
Set whether the context outputs the input embeddings of a specific layer
5286+
"""
5287+
...
5288+
5289+
# // mirrors:
5290+
# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
5291+
# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid);
5292+
@ctypes_function_llama_ext(
5293+
[
5294+
"llama_get_embeddings_layer_inp",
5295+
"?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z",
5296+
"__Z30llama_get_embeddings_layer_inpP13llama_contextj",
5297+
],
5298+
[llama_context_p_ctypes, ctypes.c_int32],
5299+
ctypes.POINTER(ctypes.c_float),
5300+
required=False,
5301+
)
5302+
def llama_get_embeddings_layer_inp(
5303+
ctx: llama_context_p,
5304+
lid: ctypes.c_int32,
5305+
) -> ctypes.POINTER(ctypes.c_float): # type: ignore
5306+
...
5307+
5308+
# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx);
5309+
@ctypes_function_llama_ext(
5310+
[
5311+
"llama_get_ctx_other",
5312+
"?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z",
5313+
"__Z19llama_get_ctx_otherP13llama_context",
5314+
],
5315+
[llama_context_p_ctypes],
5316+
llama_context_p_ctypes,
5317+
required=False,
5318+
)
5319+
def llama_get_ctx_other(
5320+
ctx: llama_context_p,
5321+
) -> llama_context_p:
5322+
...
5323+
5324+
# // model/context data extraction
5325+
5326+
# // returns pointer to the target-model layer indices
5327+
# LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model);
5328+
@ctypes_function_llama_ext(
5329+
[
5330+
"llama_model_target_layer_ids",
5331+
"?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z",
5332+
"__Z28llama_model_target_layer_idsPK11llama_model",
5333+
],
5334+
[llama_model_p_ctypes],
5335+
ctypes.POINTER(ctypes.c_int32),
5336+
required=False,
5337+
)
5338+
def llama_model_target_layer_ids(
5339+
model: llama_model_p
5340+
) -> ctypes.POINTER(ctypes.c_int32): # type: ignore
5341+
"""
5342+
returns pointer to the target-model layer indices
5343+
"""
5344+
...
5345+
5346+
# // returns the number of extracted layers from target model
5347+
# LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model);
5348+
@ctypes_function_llama_ext(
5349+
[
5350+
"llama_model_target_layer_ids_n",
5351+
"?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z",
5352+
"__Z30llama_model_target_layer_ids_nPK11llama_model"
5353+
],
5354+
[llama_model_p_ctypes],
5355+
ctypes.POINTER(ctypes.c_uint32),
5356+
required=False,
5357+
)
5358+
def llama_model_target_layer_ids_n(
5359+
model: llama_model_p
5360+
) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore
5361+
"""
5362+
returns the number of extracted layers from target model
5363+
"""
5364+
...

0 commit comments

Comments
 (0)