feat: add ds4 backend (DeepSeek V4 Flash) with tool calls, thinking, KV cache

.agents/ds4-backend.md

@@ -0,0 +1,80 @@
+# Working on the ds4 Backend
+
+`antirez/ds4` is a single-model inference engine for DeepSeek V4 Flash.
+LocalAI wraps the engine's C API (`ds4/ds4.h`) with a fresh C++ gRPC server at
+`backend/cpp/ds4/` - NOT a fork of llama-cpp's grpc-server.cpp.
+
+## Pin
+
+`backend/cpp/ds4/prepare.sh` clones `antirez/ds4` at `DS4_VERSION`. Bump that
+commit to follow upstream.
+
+## Wire shape
+
+| RPC | Implementation |
+|---|---|
+| Health, Free, Status | Trivial; no engine dependency for Health |
+| LoadModel | `ds4_engine_open` + `ds4_session_create`; backend is compile-time (DS4_NO_GPU → CPU, __APPLE__ → Metal, otherwise CUDA) |
+| TokenizeString | `ds4_tokenize_text` |
+| Predict | `ds4_engine_generate_argmax` + `DsmlParser` → one ChatDelta with content / reasoning_content / tool_calls[] |
+| PredictStream | Same, per-token ChatDelta writes |
+
+## DSML
+
+ds4 emits tool calls as literal text markers (`<｜DSML｜tool_calls>` etc.) -
+NOT special tokens. `dsml_parser.{h,cpp}` is our streaming state machine that
+classifies token bytes into CONTENT / REASONING / TOOL_START / TOOL_ARGS / TOOL_END
+events. `dsml_renderer.{h,cpp}` does the prompt direction: turns
+OpenAI tool_calls + role=tool messages back into DSML for the next turn.
+
+## Thinking modes
+
+`PredictOptions.Metadata["enable_thinking"]` gates thinking on/off (default ON).
+`["reasoning_effort"] == "max" | "xhigh"` selects `DS4_THINK_MAX`; anything else
+maps to `DS4_THINK_HIGH`. We pass the chosen mode to `ds4_chat_append_assistant_prefix`.
+
+## Disk KV cache
+
+`kv_cache.{h,cpp}` implements an SHA1-keyed file cache using ds4's public
+`ds4_session_save_payload` / `ds4_session_load_payload` API. Enable per request
+via `ModelOptions.Options[] = "kv_cache_dir:/some/path"`. Format is **our own** -
+NOT bit-compatible with ds4-server's KVC files (interop is a follow-up plan).
+
+## Build matrix
+
+| Build | Where | Notes |
+|---|---|---|
+| `cpu-ds4` (amd64 + arm64) | Linux GHA | ds4 considers CPU debug-only; useful only for wiring tests |
+| `cuda13-ds4` (amd64 + arm64) | Linux GHA + DGX Spark validation | Primary production path on Linux |
+| `ds4-darwin` (arm64) | macOS GHA runners | Metal; uses `scripts/build/ds4-darwin.sh` like llama-cpp-darwin |
+
+cuda12 is intentionally omitted. ROCm / Vulkan / SYCL are not applicable.
+
+## Hardware-gated validation
+
+`tests/e2e-backends/backend_test.go` in `BACKEND_BINARY` mode:
+
+```
+BACKEND_BINARY=$(pwd)/backend/cpp/ds4/package/run.sh \
+BACKEND_TEST_MODEL_FILE=/path/to/ds4flash.gguf \
+BACKEND_TEST_CAPS=health,load,predict,stream,tools \
+BACKEND_TEST_TOOL_PROMPT="What's the weather in Paris?" \
+go test -count=1 -timeout=30m -v ./tests/e2e-backends/...
+```
+
+CI does not load the model; the suite is opt-in via env vars.
+
+## Importer
+
+`core/gallery/importers/ds4.go` (`DS4Importer`) auto-detects ds4 weights by
+matching the `antirez/deepseek-v4-gguf` repo URI or the
+`DeepSeek-V4-Flash-*.gguf` filename pattern. **Registered BEFORE
+`LlamaCPPImporter`** in `defaultImporters` - both match `.gguf` but ds4 is more
+specific, and first-match-wins. The importer emits `backend: ds4`, uses
+`ds4flash.gguf` as the local filename (matches ds4's own CLI default), and
+disables the Go-side automatic tool-parsing fallback (the C++ backend emits
+ChatDelta.tool_calls natively via `DsmlParser`).
+
+ds4 is also listed in `core/http/endpoints/localai/backend.go`'s pref-only
+slice so the `/import-model` UI surfaces it as a manual choice for users who
+want to force the backend on a non-canonical URI.

.github/backend-matrix.yml

@@ -948,6 +948,32 @@ include:
     backend: "turboquant"
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-ds4'
+    runs-on: 'ubuntu-latest'
+    base-image: "nvidia/cuda:13.0.0-devel-ubuntu24.04"
+    skip-drivers: 'true'
+    backend: "ds4"
+    dockerfile: "./backend/Dockerfile.ds4"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'true'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-ds4'
+    base-image: "nvidia/cuda:13.0.0-devel-ubuntu24.04"
+    runs-on: 'ubuntu-24.04-arm'
+    ubuntu-version: '2404'
+    backend: "ds4"
+    dockerfile: "./backend/Dockerfile.ds4"
+    context: "./"
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -2321,6 +2347,34 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-ds4'
+    runs-on: 'ubuntu-latest'
+    base-image: "nvidia/cuda:13.0.0-devel-ubuntu24.04"
+    skip-drivers: 'true'
+    backend: "ds4"
+    dockerfile: "./backend/Dockerfile.ds4"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-ds4'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "nvidia/cuda:13.0.0-devel-ubuntu24.04"
+    skip-drivers: 'true'
+    backend: "ds4"
+    dockerfile: "./backend/Dockerfile.ds4"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""

.github/workflows/backend_build_darwin.yml

@@ -211,8 +211,13 @@ jobs:
           make protogen-go
           make backends/llama-cpp-darwin
 
+      - name: Build ds4 backend (Darwin Metal)
+        if: inputs.backend == 'ds4'
+        run: |
+          make backends/ds4-darwin
+
       - name: Build ${{ inputs.backend }}-darwin
-        if: inputs.backend != 'llama-cpp'
+        if: inputs.backend != 'llama-cpp' && inputs.backend != 'ds4'
         run: |
           make protogen-go
           BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend

AGENTS.md

@@ -25,6 +25,7 @@ LocalAI follows the Linux kernel project's [guidelines for AI coding assistants]
 | [.agents/llama-cpp-backend.md](.agents/llama-cpp-backend.md) | Working on the llama.cpp backend — architecture, updating, tool call parsing |
 | [.agents/vllm-backend.md](.agents/vllm-backend.md) | Working on the vLLM / vLLM-omni backends — native parsers, ChatDelta, CPU build, libnuma packaging, backend hooks |
 | [.agents/sglang-backend.md](.agents/sglang-backend.md) | Working on the SGLang backend — `engine_args` validation against ServerArgs, speculative-decoding (EAGLE/EAGLE3/DFLASH/MTP) recipes, parser handling |
+| [.agents/ds4-backend.md](.agents/ds4-backend.md) | Working on the ds4 backend - DSML state machine, thinking modes, KV cache, Metal+CUDA matrix |
 | [.agents/testing-mcp-apps.md](.agents/testing-mcp-apps.md) | Testing MCP Apps (interactive tool UIs) in the React UI |
 | [.agents/api-endpoints-and-auth.md](.agents/api-endpoints-and-auth.md) | Adding API endpoints, auth middleware, feature permissions, user access control |
 | [.agents/debugging-backends.md](.agents/debugging-backends.md) | Debugging runtime backend failures, dependency conflicts, rebuilding backends |

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -1009,6 +1009,10 @@ backends/llama-cpp-darwin: build
 	bash ./scripts/build/llama-cpp-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)"
 
+backends/ds4-darwin: build
+	bash ./scripts/build/ds4-darwin.sh
+	./local-ai backends install "ocifile://$(abspath ./backend-images/ds4.tar)"
+
 build-darwin-python-backend: build
 	bash ./scripts/build/python-darwin.sh
 
@@ -1050,6 +1054,10 @@ BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 # turboquant is a llama.cpp fork with TurboQuant KV-cache quantization.
 # Reuses backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
 BACKEND_TURBOQUANT = turboquant|turboquant|.|false|false
+# ds4 is antirez/ds4, a DeepSeek V4 Flash-specific inference engine.
+# Single-model; hardware-only validation lives at tests/e2e-backends/
+# (BACKEND_BINARY mode); see docs/superpowers/plans/2026-05-11-ds4-backend.md.
+BACKEND_DS4 = ds4|ds4|.|false|false
 
 # Golang backends
 BACKEND_PIPER = piper|golang|.|false|true
@@ -1135,6 +1143,7 @@ endef
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TURBOQUANT)))
+$(eval $(call generate-docker-build-target,$(BACKEND_DS4)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
@@ -1188,7 +1197,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SHERPA_ONNX)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx
 
 ########################################################
 ### Mock Backend for E2E Tests

backend/Dockerfile.ds4

@@ -0,0 +1,41 @@
+ARG BASE_IMAGE=ubuntu:24.04
+ARG APT_MIRROR=""
+ARG APT_PORTS_MIRROR=""
+
+# BASE_IMAGE is either ubuntu:24.04 (for cpu builds) or nvidia/cuda:13.0.0-devel-ubuntu24.04
+# (for cublas builds). Both ship apt + Ubuntu Noble packages; the nvidia/cuda base
+# additionally provides /usr/local/cuda. Darwin (Metal) builds bypass this Dockerfile
+# entirely via scripts/build/ds4-darwin.sh.
+FROM ${BASE_IMAGE} AS builder
+ARG BUILD_TYPE
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+ENV BUILD_TYPE=${BUILD_TYPE} \
+    DEBIAN_FRONTEND=noninteractive \
+    PATH=/usr/local/cuda/bin:${PATH}
+
+WORKDIR /build
+
+# Install build-time deps via plain apt - install-base-deps.sh's full pipeline
+# (CUDA keyring + from-source gRPC) is unnecessary here:
+#   - CUDA: when BASE_IMAGE=nvidia/cuda:*, /usr/local/cuda is already populated;
+#     for the cpu build we don't need CUDA at all.
+#   - gRPC/Protobuf: system apt packages are sufficient; ds4's wrapper only links
+#     against them, it doesn't ship the gRPC source tree.
+#   - nlohmann-json: dsml_renderer's only third-party dep.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git cmake build-essential pkg-config ca-certificates \
+        libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc \
+        nlohmann-json3-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . /LocalAI
+
+RUN --mount=type=cache,target=/root/.ccache,id=ds4-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package
+
+FROM scratch
+COPY --from=builder /LocalAI/backend/cpp/ds4/package/. ./

backend/cpp/ds4/.gitignore

@@ -0,0 +1,9 @@
+ds4/
+build/
+package/
+grpc-server
+*.o
+backend.pb.cc
+backend.pb.h
+backend.grpc.pb.cc
+backend.grpc.pb.h

backend/cpp/ds4/CMakeLists.txt

@@ -0,0 +1,101 @@
+cmake_minimum_required(VERSION 3.15)
+project(ds4-grpc-server LANGUAGES CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(TARGET grpc-server)
+
+option(DS4_NATIVE "Compile with -march=native / -mcpu=native" ON)
+set(DS4_GPU "cpu" CACHE STRING "GPU backend: cpu, cuda, or metal")
+set(DS4_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ds4" CACHE PATH "Path to cloned ds4 source")
+
+find_package(Threads REQUIRED)
+find_package(Protobuf CONFIG QUIET)
+if(NOT Protobuf_FOUND)
+    find_package(Protobuf REQUIRED)
+endif()
+find_package(gRPC CONFIG QUIET)
+if(NOT gRPC_FOUND)
+    # Ubuntu's apt-installed grpc++ does not ship a CMake config - fall back.
+    find_library(GRPCPP_LIB grpc++ REQUIRED)
+    find_library(GRPCPP_REFLECTION_LIB grpc++_reflection REQUIRED)
+    add_library(gRPC::grpc++ INTERFACE IMPORTED)
+    set_target_properties(gRPC::grpc++ PROPERTIES INTERFACE_LINK_LIBRARIES "${GRPCPP_LIB}")
+    add_library(gRPC::grpc++_reflection INTERFACE IMPORTED)
+    set_target_properties(gRPC::grpc++_reflection PROPERTIES INTERFACE_LINK_LIBRARIES "${GRPCPP_REFLECTION_LIB}")
+endif()
+
+find_program(_PROTOC NAMES protoc REQUIRED)
+find_program(_GRPC_CPP_PLUGIN NAMES grpc_cpp_plugin REQUIRED)
+
+get_filename_component(HW_PROTO "${CMAKE_CURRENT_SOURCE_DIR}/../../backend.proto" ABSOLUTE)
+get_filename_component(HW_PROTO_PATH "${HW_PROTO}" PATH)
+
+set(HW_PROTO_SRCS "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.cc")
+set(HW_PROTO_HDRS "${CMAKE_CURRENT_BINARY_DIR}/backend.pb.h")
+set(HW_GRPC_SRCS  "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.cc")
+set(HW_GRPC_HDRS  "${CMAKE_CURRENT_BINARY_DIR}/backend.grpc.pb.h")
+
+add_custom_command(
+    OUTPUT "${HW_PROTO_SRCS}" "${HW_PROTO_HDRS}" "${HW_GRPC_SRCS}" "${HW_GRPC_HDRS}"
+    COMMAND ${_PROTOC}
+    ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+         --cpp_out  "${CMAKE_CURRENT_BINARY_DIR}"
+         -I "${HW_PROTO_PATH}"
+         --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN}"
+         "${HW_PROTO}"
+    DEPENDS "${HW_PROTO}")
+
+add_library(hw_grpc_proto STATIC
+    ${HW_GRPC_SRCS} ${HW_GRPC_HDRS}
+    ${HW_PROTO_SRCS} ${HW_PROTO_HDRS})
+target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+set(DS4_OBJS "${DS4_DIR}/ds4.o")
+if(DS4_GPU STREQUAL "cuda")
+    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_cuda.o")
+elseif(DS4_GPU STREQUAL "metal")
+    list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o")
+elseif(DS4_GPU STREQUAL "cpu")
+    set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o")
+endif()
+
+add_executable(${TARGET}
+    grpc-server.cpp
+    dsml_parser.cpp
+    dsml_renderer.cpp
+    kv_cache.cpp)
+
+target_include_directories(${TARGET} PRIVATE ${DS4_DIR})
+
+foreach(obj ${DS4_OBJS})
+    target_sources(${TARGET} PRIVATE ${obj})
+    set_source_files_properties(${obj} PROPERTIES EXTERNAL_OBJECT TRUE GENERATED TRUE)
+endforeach()
+
+target_link_libraries(${TARGET} PRIVATE
+    hw_grpc_proto
+    gRPC::grpc++
+    gRPC::grpc++_reflection
+    protobuf::libprotobuf
+    Threads::Threads
+    m)
+
+if(DS4_GPU STREQUAL "cuda")
+    find_package(CUDAToolkit REQUIRED)
+    target_link_libraries(${TARGET} PRIVATE CUDA::cudart CUDA::cublas)
+elseif(DS4_GPU STREQUAL "metal")
+    find_library(FOUNDATION_LIB Foundation REQUIRED)
+    find_library(METAL_LIB Metal REQUIRED)
+    target_link_libraries(${TARGET} PRIVATE ${FOUNDATION_LIB} ${METAL_LIB})
+elseif(DS4_GPU STREQUAL "cpu")
+    target_compile_definitions(${TARGET} PRIVATE DS4_NO_GPU)
+endif()
+
+if(DS4_NATIVE)
+    if(APPLE)
+        target_compile_options(${TARGET} PRIVATE -mcpu=native)
+    else()
+        target_compile_options(${TARGET} PRIVATE -march=native)
+    endif()
+endif()