feat(backend): add buun-llama-cpp fork (DFlash + TCQ KV-cache)

.agents/ai-coding-assistants.md

@@ -35,19 +35,33 @@ All contributions must comply with LocalAI's licensing requirements:
 
 ## Signed-off-by and Developer Certificate of Origin
 
-**AI agents MUST NOT add `Signed-off-by` tags.** Only humans can legally
-certify the Developer Certificate of Origin (DCO). The human submitter
-is responsible for:
-
-- Reviewing all AI-generated code
+Only humans can certify the Developer Certificate of Origin (DCO). AI
+agents MUST NOT invent or guess a human identity for `Signed-off-by` —
+doing so forges the DCO certification.
+
+However, when a human operator explicitly directs the AI to commit on
+their behalf, the AI is acting as a typing tool — no different from an
+editor macro or `git commit -s`. In that case the AI SHOULD add
+`Signed-off-by:` using the **configured `user.name` / `user.email`** of
+the current git repository (i.e. the operator's own identity). The
+resulting trailer is the operator's signature; they take responsibility
+for it by reviewing and pushing the commit. The AI MUST NOT use any
+other identity and MUST NOT add its own name to the sign-off.
+
+When running `git commit`, prefer `git commit --signoff` (or `-s`) so
+the trailer is emitted by git itself from the configured identity,
+rather than hand-writing it in a heredoc — this guarantees the sign-off
+matches whatever identity the operator is currently using.
+
+The human submitter remains responsible for:
+
+- Reviewing all AI-generated code before it's pushed or merged
 - Ensuring compliance with licensing requirements
-- Adding their own `Signed-off-by` tag (when the project requires DCO)
-  to certify the contribution
 - Taking full responsibility for the contribution
 
-AI agents MUST NOT add `Co-Authored-By` trailers for themselves either.
-A human reviewer owns the contribution; the AI's involvement is recorded
-via `Assisted-by` (see below).
+AI agents MUST NOT add `Co-Authored-By` trailers for themselves. A human
+reviewer owns the contribution; the AI's involvement is recorded via
+`Assisted-by` (see below).
 
 ## Attribution
 
@@ -84,6 +98,12 @@ Assisted-by: Claude:claude-opus-4-7 golangci-lint
 Signed-off-by: Jane Developer <jane@example.com>
 ```
 
+The `Signed-off-by` line uses Jane's own identity because Jane is the
+submitter operating the AI. If Jane asks Claude to create the commit via
+`git commit -s`, git emits that exact trailer from Jane's configured
+identity — no separate human step is needed beyond Jane reviewing the
+diff before pushing.
+
 ## Scope and Responsibility
 
 Using an AI assistant does not reduce the contributor's responsibility.

.github/backend-matrix.yml

@@ -480,6 +480,22 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-cuda-12-amd64'
+    # bigger-runner: same rationale as -gpu-nvidia-cuda-12-llama-cpp above
+    # (observed 6h5m wall-clock on v4.2.1, just past the 6h job timeout).
+    runs-on: 'bigger-runner'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1165,6 +1181,21 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-cuda-13-amd64'
+    # bigger-runner: observed 6h5m wall-clock on v4.2.1 — at the GHA timeout.
+    runs-on: 'bigger-runner'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1208,6 +1239,20 @@ include:
     backend: "turboquant"
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-cuda-13-arm64'
+    base-image: "ubuntu:24.04"
+    runs-on: 'ubuntu-24.04-arm'
+    ubuntu-version: '2404'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -2477,6 +2522,20 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'sycl_f32'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f32-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-intel-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'sycl_f32'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2519,6 +2578,20 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'sycl_f16'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-intel-sycl-f16-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-intel-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'sycl_f16'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -2985,6 +3058,21 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3015,6 +3103,21 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3276,6 +3379,20 @@ include:
     dockerfile: "./backend/Dockerfile.turboquant"
     context: "./"
     ubuntu-version: '2204'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    skip-drivers: 'false'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-arm64-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-l4t-cuda-12-arm64'
+    base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
+    runs-on: 'ubuntu-24.04-arm'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2204'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "0"
@@ -3336,6 +3453,22 @@ include:
     context: "./"
     ubuntu-version: '2404'
   # Stablediffusion-ggml
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
+  # Stablediffusion-ggml
   - build-type: 'vulkan'
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -3368,6 +3501,22 @@ include:
     context: "./"
     ubuntu-version: '2404'
   # Stablediffusion-ggml
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-buun-llama-cpp'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "buun-llama-cpp"
+    dockerfile: "./backend/Dockerfile.buun-llama-cpp"
+    context: "./"
+    ubuntu-version: '2404'
+  # Stablediffusion-ggml
   - build-type: 'vulkan'
     cuda-major-version: ""
     cuda-minor-version: ""

.github/workflows/test-extra.yml

@@ -33,6 +33,7 @@ jobs:
       llama-cpp: ${{ steps.detect.outputs.llama-cpp }}
       ik-llama-cpp: ${{ steps.detect.outputs.ik-llama-cpp }}
       turboquant: ${{ steps.detect.outputs.turboquant }}
+      buun-llama-cpp: ${{ steps.detect.outputs['buun-llama-cpp'] }}
       vllm: ${{ steps.detect.outputs.vllm }}
       sglang: ${{ steps.detect.outputs.sglang }}
       acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
@@ -716,6 +717,30 @@ jobs:
       - name: Build turboquant backend image and run gRPC e2e tests
         run: |
           make test-extra-backend-turboquant
+  tests-buun-llama-cpp-grpc:
+    needs: detect-changes
+    if: needs.detect-changes.outputs['buun-llama-cpp'] == 'true' || needs.detect-changes.outputs.run-all == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Clone
+        uses: actions/checkout@v6
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.25.4'
+      # Exercises the buun-llama-cpp (fork-of-a-fork) backend with the
+      # fork-specific TurboQuant/TCQ KV-cache types. BACKEND_TEST_CACHE_TYPE_V
+      # is set to turbo3 so the test round-trips through the fork's KV
+      # allow-list — picking a stock llama.cpp type would only re-test the
+      # shared code path. DFlash speculative decoding is not exercised here
+      # because the one known public target/drafter pair (Qwen3.5-27B) is too
+      # large for CI.
+      - name: Build buun-llama-cpp backend image and run gRPC e2e tests
+        run: |
+          make test-extra-backend-buun-llama-cpp
   # tests-vllm-grpc is currently disabled in CI.
   #
   # The prebuilt vllm CPU wheel is compiled with AVX-512 VNNI/BF16

Makefile

@@ -1,4 +1,5 @@
 # Disable parallel execution for backend builds
+.NOTPARALLEL: backends/buun-llama-cpp
 .NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 
 GOCMD=go
@@ -748,6 +749,19 @@ test-extra-backend-bonsai: docker-build-bonsai
 	BACKEND_TEST_MODEL_URL=https://huggingface.co/prism-ml/Bonsai-8B-gguf/resolve/main/Bonsai-8B-Q1_0.gguf \
 	$(MAKE) test-extra-backend
 
+## buun-llama-cpp: exercises the fork-of-a-fork backend (spiritbuun/buun-llama-cpp)
+## with the *TurboQuant/TCQ-specific* KV-cache types (turbo3 for V). Same rationale
+## as turboquant above: picking a standard llama.cpp type would only re-test the
+## shared code path. buun inherits turboquant's turbo2/turbo3/turbo4 and adds
+## turbo2_tcq / turbo3_tcq on top. DFlash speculative decoding is not exercised
+## here because no small DFlash drafter model exists (the known public pair is
+## Qwen3.5-27B, ~54 GB).
+test-extra-backend-buun-llama-cpp: docker-build-buun-llama-cpp
+	BACKEND_IMAGE=local-ai-backend:buun-llama-cpp \
+	BACKEND_TEST_CACHE_TYPE_K=q8_0 \
+	BACKEND_TEST_CACHE_TYPE_V=turbo3 \
+	$(MAKE) test-extra-backend
+
 ## Audio transcription wrapper for the llama-cpp backend.
 ## Drives the new AudioTranscription / AudioTranscriptionStream RPCs against
 ## ggml-org/Qwen3-ASR-0.6B-GGUF (a small ASR model that requires its mmproj
@@ -1284,6 +1298,11 @@ BACKEND_PRIVACY_FILTER = privacy-filter|privacy-filter|.|false|false
 # against apt gRPC/protobuf rather than a prebuilt base-grpc image; the reason
 # is on the audio-cpp block in .github/backend-matrix.yml.
 BACKEND_AUDIO_CPP = audio-cpp|audio-cpp|.|false|false
+# buun-llama-cpp is a fork-of-a-fork (spiritbuun/buun-llama-cpp forks
+# TheTom/llama-cpp-turboquant) that adds DFlash block-diffusion speculative
+# decoding and extra TCQ KV-cache variants on top of TurboQuant. Same thin
+# wrapper pattern as turboquant — reuses backend/cpp/llama-cpp grpc-server.
+BACKEND_BUUN_LLAMA_CPP = buun-llama-cpp|buun-llama-cpp|.|false|false
 
 # Golang backends
 BACKEND_PIPER = piper|golang|.|false|true
@@ -1388,6 +1407,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_BONSAI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DS4)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_AUDIO_CPP)))
+$(eval $(call generate-docker-build-target,$(BACKEND_BUUN_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
 $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VALKEY_STORE)))
@@ -1456,7 +1476,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-buun-llama-cpp docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp
 
 ########################################################
 ### Mock Backend for E2E Tests