Update llama.cpp API 20260108

Author: JamePeng

examples/low_level_api/common.py

@@ -62,6 +62,7 @@ class GptParams:
     penalize_nl: bool = True
     perplexity: bool = False
     use_mmap: bool = True
+    use_direct_io: bool = True
     use_mlock: bool = False
     mem_test: bool = False
     verbose_prompt: bool = False

examples/low_level_api/low_level_api_chat_cpp.py

@@ -78,6 +78,7 @@ def __init__(self, params: GptParams) -> None:
         self.lparams.memory_f16 = self.params.memory_f16
         self.lparams.use_mlock = self.params.use_mlock
         self.lparams.use_mmap = self.params.use_mmap
+        self.lparams.use_direct_io = self.params.use_direct_io
 
         self.model = llama_cpp.llama_load_model_from_file(
             self.params.model.encode("utf8"), self.lparams

llama_cpp/llama.py

@@ -69,6 +69,7 @@ def __init__(
         tensor_split: Optional[List[float]] = None,
         vocab_only: bool = False,
         use_mmap: bool = True,
+        use_direct_io: bool = True,
         use_mlock: bool = False,
         check_tensors: bool = False,
         use_extra_bufts: bool = False,
@@ -256,6 +257,7 @@ def __init__(
             self.model_params.tensor_split = self._c_tensor_split
         self.model_params.vocab_only = vocab_only
         self.model_params.use_mmap = use_mmap if lora_path is None else False
+        self.model_params.use_direct_io = use_direct_io
         self.model_params.use_mlock = use_mlock
         self.model_params.check_tensors = check_tensors
         self.model_params.use_extra_bufts = use_extra_bufts
@@ -2248,6 +2250,7 @@ def __getstate__(self):
             tensor_split=self.tensor_split,
             vocab_only=self.model_params.vocab_only,
             use_mmap=self.model_params.use_mmap,
+            use_direct_io=self.model_params.use_direct_io,
             use_mlock=self.model_params.use_mlock,
             check_tensors=self.model_params.check_tensors,
             use_extra_bufts=self.model_params.use_extra_bufts,

llama_cpp/llama_cpp.py

@@ -713,6 +713,7 @@ class llama_model_tensor_buft_override(ctypes.Structure):
 #     // Keep the booleans together to avoid misalignment during copy-by-value.
 #     bool vocab_only;      // only load the vocabulary, no weights
 #     bool use_mmap;        // use mmap if possible
+#     bool use_direct_io;   // use direct io, takes precedence over use_mmap
 #     bool use_mlock;       // force system to keep model in RAM
 #     bool check_tensors;   // validate model tensor data
 #     bool use_extra_bufts; // use extra buffer types (used for weight repacking)
@@ -734,6 +735,7 @@ class llama_model_params(ctypes.Structure):
         kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data
         vocab_only (bool): only load the vocabulary, no weights
         use_mmap (bool): use mmap if possible
+        use_direct_io(bool): use direct io, takes precedence over use_mmap
         use_mlock (bool): force system to keep model in RAM
         check_tensors (bool): validate model tensor data
         use_extra_bufts (bool): use extra buffer types (used for weight repacking)
@@ -752,6 +754,7 @@ class llama_model_params(ctypes.Structure):
         kv_overrides: CtypesArray[llama_model_kv_override]
         vocab_only: bool
         use_mmap: bool
+        use_direct_io: bool
         use_mlock: bool
         check_tensors: bool
         use_extra_bufts: bool
@@ -770,6 +773,7 @@ class llama_model_params(ctypes.Structure):
         ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),
         ("vocab_only", ctypes.c_bool),
         ("use_mmap", ctypes.c_bool),
+        ("use_direct_io", ctypes.c_bool),
         ("use_mlock", ctypes.c_bool),
         ("check_tensors", ctypes.c_bool),
         ("use_extra_bufts", ctypes.c_bool),

llama_cpp/server/model.py

@@ -270,7 +270,11 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
             tensor_split=settings.tensor_split,
             vocab_only=settings.vocab_only,
             use_mmap=settings.use_mmap,
+            use_direct_io=settings.use_direct_io,
             use_mlock=settings.use_mlock,
+            check_tensors=settings.check_tensors,
+            use_extra_bufts=settings.use_extra_bufts,
+            no_host=settings.no_host,
             kv_overrides=kv_overrides,
             rpc_servers=settings.rpc_servers,
             # Context Params

llama_cpp/server/settings.py

@@ -47,12 +47,28 @@ class ModelSettings(BaseSettings):
         default=False, description="Whether to only return the vocabulary."
     )
     use_mmap: bool = Field(
-        default=llama_cpp.llama_supports_mmap(),
-        description="Use mmap.",
+        default=True,
+        description="Enable mmap to use filesystem cache.",
+    )
+    use_direct_io: bool = Field(
+        default=True,
+        description="Use direct io, takes precedence over use_mmap.",
     )
     use_mlock: bool = Field(
-        default=llama_cpp.llama_supports_mlock(),
-        description="Use mlock.",
+        default=False,
+        description="Use mlock for force system to keep model in RAM",
+    )
+    check_tensors: bool = Field(
+        default=False,
+        description="Validate model tensor data.",
+    )
+    use_extra_bufts: bool = Field(
+        default=True,
+        description="Use extra buffer types (used for weight repacking).",
+    )
+    no_host: bool = Field(
+        default=False,
+        description="Bypass host buffer allowing extra buffers to be used.",
     )
     kv_overrides: Optional[List[str]] = Field(
         default=None,

tests/test_llama.py

@@ -75,6 +75,7 @@ def test_real_model(llama_cpp_model_path):
 
     params = llama_cpp.llama_model_default_params()
     params.use_mmap = llama_cpp.llama_supports_mmap()
+    params.use_direct_io = True
     params.use_mlock = llama_cpp.llama_supports_mlock()
     params.check_tensors = False