Merge branch 'master' into feat/vllm-cpp-darwin-mlx

Author: localai-org-maint-bot

.github/backend-matrix.yml

@@ -66,6 +66,34 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-kokoro'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'true'
+    backend: "kokoro"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-kokoro'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'true'
+    backend: "kokoro"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""

.github/workflows/image-pr.yml

@@ -52,7 +52,7 @@
               tag-latest: 'false'
               tag-suffix: '-gpu-nvidia-cuda-13'
               runs-on: 'ubuntu-latest'
-              base-image: "ubuntu:22.04"
+              base-image: "ubuntu:24.04"
               makeflags: "--jobs=3 --output-sync=target"
               ubuntu-version: '2404'
             - build-type: 'hipblas'

.github/workflows/image.yml

@@ -113,7 +113,7 @@
               tag-latest: 'auto'
               tag-suffix: '-gpu-nvidia-cuda-13'
               runs-on: 'ubuntu-latest'
-              base-image: "ubuntu:22.04"
+              base-image: "ubuntu:24.04"
               skip-drivers: 'false'
               makeflags: "--jobs=4 --output-sync=target"
               ubuntu-version: '2404'

backend/Dockerfile.golang

@@ -221,6 +221,33 @@ RUN if [ "${BACKEND}" = "crispasr" ]; then \
     apt-get clean && rm -rf /var/lib/apt/lists/*; \
 fi
 
+# sherpa-onnx links onnxruntime's CUDA execution provider, and
+# libonnxruntime_providers_cuda.so has cuDNN as a hard DT_NEEDED. The
+# onnxruntime GPU tarball does not ship cuDNN itself, so without this the
+# builder has none (the arm64 + CUDA 13 branch above is the only other place
+# that installs it) and package-gpu-libs.sh correctly refuses to produce a
+# package that references cuDNN with no cuDNN available to it.
+#
+# Installed per-backend rather than for every cublas build: the auto-detection
+# in package-gpu-libs.sh bundles only what a package actually references, so
+# the ggml backends would not grow either way, but they would all pay ~1.1 GB
+# of builder layer and registry cache for a library they never call.
+#
+# Runtime package only, no -dev: sherpa-onnx consumes onnxruntime's prebuilt
+# CUDA provider and never compiles against cuDNN headers. libcudnn9-cuda-N
+# carries the dispatcher plus all seven dlopen()ed sublibraries, which is what
+# complete_cudnn_family needs to assemble a whole bundle.
+RUN <<EOT bash
+    if [ "${BACKEND}" = "sherpa-onnx" ] && [ "${BUILD_TYPE}" = "cublas" ] && [ "${SKIP_DRIVERS}" = "false" ]; then
+        apt-get update && \
+        apt-get install -y --no-install-recommends \
+            libcudnn9-cuda-${CUDA_MAJOR_VERSION} && \
+        ldconfig && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+    fi
+EOT
+
 COPY . /LocalAI
 
 RUN git config --global --add safe.directory /LocalAI

backend/backend.proto

@@ -508,6 +508,12 @@ message ProxyOptions {
   string api_key_file = 5;
   string upstream_model = 6;
   int32 request_timeout_seconds = 7;
+  // cache_prompt enables automatic Anthropic prompt-cache breakpoints
+  // (cache_control: ephemeral) on the stable prefix — system, tools, and
+  // the last message block — when translating to the Anthropic provider.
+  // Cuts input cost on repeated/agentic calls (cache read = 0.1x). Only
+  // meaningful for mode=translate + provider=anthropic; ignored otherwise.
+  bool cache_prompt = 8;
 }
 
 message Result {

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=0a4e10c7fb65d2dd5a4afb78339c7d373a8cdfaa
+IK_LLAMA_VERSION?=5f063b7bbae8f9a34dfc5c704aa77939e76494a9
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=571d0d540df04f25298d0e159e520d9fc62ed121
+LLAMA_VERSION?=1cbfd1988311775425d36c0ce066590f7d3049cf
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -52,6 +52,7 @@
 #include "common.h"
 #include "arg.h"
 #include "chat-auto-parser.h"
+#include "llama_compat.h" // fork-skew switches, generated by prepare.sh
 #include "message_content.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
@@ -613,6 +614,13 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
     // starts with '-'. Applied once after the loop via common_params_parse.
     std::vector<std::string> extra_argv;
 
+    // O_DIRECT intent from the `direct_io` option. Upstream folded
+    // use_mmap/use_mlock/use_direct_io into a single common_params::load_mode
+    // enum (ggml-org/llama.cpp#20834), so the three independent LocalAI settings
+    // can only be reduced to one value once all of them have been read, held
+    // here until the mmap/mlock fields arrive further down.
+    bool want_direct_io = false;
+
     auto add_device_options = [&](const std::string & devices) {
         const std::regex regex{ R"([,]+)" };
         std::sregex_token_iterator it{ devices.begin(), devices.end(), regex, -1 };
@@ -868,9 +876,9 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
         // --- O_DIRECT model loading (upstream --direct-io) ---
         } else if (!strcmp(optname, "direct_io") || !strcmp(optname, "use_direct_io")) {
             if (optval_str == "true" || optval_str == "1" || optval_str == "yes" || optval_str == "on" || optval_str == "enabled") {
-                params.use_direct_io = true;
+                want_direct_io = true;
             } else if (optval_str == "false" || optval_str == "0" || optval_str == "no" || optval_str == "off" || optval_str == "disabled") {
-                params.use_direct_io = false;
+                want_direct_io = false;
             }
 
         // --- embedding normalization (upstream --embd-normalize) ---
@@ -1278,8 +1286,28 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
      lora_info.ptr = nullptr;
      params.lora_adapters.push_back(std::move(lora_info));
     }
-    params.use_mlock = request->mlock();
-    params.use_mmap = request->mmap();
+    // LocalAI keeps mmap, mlock and direct-I/O as three independent settings,
+    // while upstream now carries a single load mode. Fold them with the
+    // precedence the separate booleans used to give: direct I/O bypasses the
+    // page cache entirely, mlock implies mmap, and everything off is a plain
+    // buffered read. Forks that branched before ggml-org/llama.cpp#20834 still
+    // expose the booleans; prepare.sh probes the checkout and sets
+    // LOCALAI_LEGACY_LOAD_MODE in the generated llama_compat.h accordingly.
+#if LOCALAI_LEGACY_LOAD_MODE
+    params.use_mlock     = request->mlock();
+    params.use_mmap      = request->mmap();
+    params.use_direct_io = want_direct_io;
+#else
+    if (want_direct_io) {
+        params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO;
+    } else if (request->mlock()) {
+        params.load_mode = LLAMA_LOAD_MODE_MLOCK;
+    } else if (request->mmap()) {
+        params.load_mode = LLAMA_LOAD_MODE_MMAP;
+    } else {
+        params.load_mode = LLAMA_LOAD_MODE_NONE;
+    }
+#endif
 
     if (request->flashattention() == "on" || request->flashattention() == "enabled") {
         params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;