Skip to content

Commit 9ca03cf

Browse files
mudlermudler
authored
feat(backends): add ik-llama-cpp (#9326)
* feat(backends): add ik-llama-cpp Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore: add grpc e2e suite, hook to CI, update README Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Apply suggestion from @mudler Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com> * Apply suggestion from @mudler Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
1 parent 151ad27 commit 9ca03cf

20 files changed

Lines changed: 4360 additions & 2 deletions

‎.github/workflows/backend.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1945,6 +1945,19 @@ jobs:
19451945
dockerfile: "./backend/Dockerfile.llama-cpp"
19461946
context: "./"
19471947
ubuntu-version: '2404'
1948+
- build-type: ''
1949+
cuda-major-version: ""
1950+
cuda-minor-version: ""
1951+
platforms: 'linux/amd64'
1952+
tag-latest: 'auto'
1953+
tag-suffix: '-cpu-ik-llama-cpp'
1954+
runs-on: 'bigger-runner'
1955+
base-image: "ubuntu:24.04"
1956+
skip-drivers: 'false'
1957+
backend: "ik-llama-cpp"
1958+
dockerfile: "./backend/Dockerfile.ik-llama-cpp"
1959+
context: "./"
1960+
ubuntu-version: '2404'
19481961
- build-type: 'cublas'
19491962
cuda-major-version: "12"
19501963
cuda-minor-version: "0"

‎.github/workflows/bump_deps.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ jobs:
1414
variable: "LLAMA_VERSION"
1515
branch: "master"
1616
file: "backend/cpp/llama-cpp/Makefile"
17+
- repository: "ikawrakow/ik_llama.cpp"
18+
variable: "IK_LLAMA_VERSION"
19+
branch: "main"
20+
file: "backend/cpp/ik-llama-cpp/Makefile"
1721
- repository: "ggml-org/whisper.cpp"
1822
variable: "WHISPER_CPP_VERSION"
1923
branch: "master"

‎.github/workflows/test-extra.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ jobs:
2929
nemo: ${{ steps.detect.outputs.nemo }}
3030
voxcpm: ${{ steps.detect.outputs.voxcpm }}
3131
llama-cpp-quantization: ${{ steps.detect.outputs.llama-cpp-quantization }}
32+
llama-cpp: ${{ steps.detect.outputs.llama-cpp }}
33+
ik-llama-cpp: ${{ steps.detect.outputs.ik-llama-cpp }}
3234
acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
3335
qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
3436
voxtral: ${{ steps.detect.outputs.voxtral }}
@@ -465,6 +467,40 @@ jobs:
465467
- name: Test llama-cpp-quantization
466468
run: |
467469
make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization test
470+
tests-llama-cpp-grpc:
471+
needs: detect-changes
472+
if: needs.detect-changes.outputs.llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
473+
runs-on: ubuntu-latest
474+
timeout-minutes: 90
475+
steps:
476+
- name: Clone
477+
uses: actions/checkout@v6
478+
with:
479+
submodules: true
480+
- name: Setup Go
481+
uses: actions/setup-go@v5
482+
with:
483+
go-version: '1.25.4'
484+
- name: Build llama-cpp backend image and run gRPC e2e tests
485+
run: |
486+
make test-extra-backend-llama-cpp
487+
tests-ik-llama-cpp-grpc:
488+
needs: detect-changes
489+
if: needs.detect-changes.outputs.ik-llama-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
490+
runs-on: ubuntu-latest
491+
timeout-minutes: 90
492+
steps:
493+
- name: Clone
494+
uses: actions/checkout@v6
495+
with:
496+
submodules: true
497+
- name: Setup Go
498+
uses: actions/setup-go@v5
499+
with:
500+
go-version: '1.25.4'
501+
- name: Build ik-llama-cpp backend image and run gRPC e2e tests
502+
run: |
503+
make test-extra-backend-ik-llama-cpp
468504
tests-acestep-cpp:
469505
needs: detect-changes
470506
if: needs.detect-changes.outputs.acestep-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'

‎Makefile‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,47 @@ test-extra: prepare-test-extra
456456
$(MAKE) -C backend/python/trl test
457457
$(MAKE) -C backend/rust/kokoros test
458458

459+
##
460+
## End-to-end gRPC tests that exercise a built backend container image.
461+
##
462+
## The test suite in tests/e2e-backends is backend-agnostic. You drive it via env
463+
## vars (see tests/e2e-backends/backend_test.go for the full list) and the
464+
## capability-driven harness picks which gRPC RPCs to exercise:
465+
##
466+
## BACKEND_IMAGE Required. Docker image to test, e.g. local-ai-backend:llama-cpp.
467+
## BACKEND_TEST_MODEL_URL URL of a model file to download and load.
468+
## BACKEND_TEST_MODEL_FILE Path to an already-downloaded model (skips download).
469+
## BACKEND_TEST_CAPS Comma-separated capabilities, default "health,load,predict,stream".
470+
## BACKEND_TEST_PROMPT Override the prompt used in predict/stream specs.
471+
##
472+
## Direct usage (image already built, no docker-build-* dependency):
473+
##
474+
## make test-extra-backend BACKEND_IMAGE=local-ai-backend:llama-cpp \
475+
## BACKEND_TEST_MODEL_URL=https://.../model.gguf
476+
##
477+
## Convenience wrappers below build a specific backend image first, then run the
478+
## suite against it.
479+
##
480+
BACKEND_TEST_MODEL_URL?=https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf
481+
482+
## Generic target — runs the suite against whatever BACKEND_IMAGE points at.
483+
## Depends on protogen-go so pkg/grpc/proto is generated before `go test`.
484+
test-extra-backend: protogen-go
485+
@test -n "$$BACKEND_IMAGE" || { echo "BACKEND_IMAGE must be set" >&2; exit 1; }
486+
BACKEND_IMAGE="$$BACKEND_IMAGE" \
487+
BACKEND_TEST_MODEL_URL="$${BACKEND_TEST_MODEL_URL:-$(BACKEND_TEST_MODEL_URL)}" \
488+
BACKEND_TEST_MODEL_FILE="$$BACKEND_TEST_MODEL_FILE" \
489+
BACKEND_TEST_CAPS="$$BACKEND_TEST_CAPS" \
490+
BACKEND_TEST_PROMPT="$$BACKEND_TEST_PROMPT" \
491+
go test -v -timeout 15m ./tests/e2e-backends/...
492+
493+
## Convenience wrappers: build the image, then exercise it.
494+
test-extra-backend-llama-cpp: docker-build-llama-cpp
495+
BACKEND_IMAGE=local-ai-backend:llama-cpp $(MAKE) test-extra-backend
496+
497+
test-extra-backend-ik-llama-cpp: docker-build-ik-llama-cpp
498+
BACKEND_IMAGE=local-ai-backend:ik-llama-cpp $(MAKE) test-extra-backend
499+
459500
DOCKER_IMAGE?=local-ai
460501
IMAGE_TYPE?=core
461502
BASE_IMAGE?=ubuntu:24.04
@@ -549,6 +590,8 @@ backend-images:
549590
# Backend metadata: BACKEND_NAME | DOCKERFILE_TYPE | BUILD_CONTEXT | PROGRESS_FLAG | NEEDS_BACKEND_ARG
550591
# llama-cpp is special - uses llama-cpp Dockerfile and doesn't need BACKEND arg
551592
BACKEND_LLAMA_CPP = llama-cpp|llama-cpp|.|false|false
593+
# ik-llama-cpp is a fork of llama.cpp with superior CPU performance
594+
BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
552595

553596
# Golang backends
554597
BACKEND_PIPER = piper|golang|.|false|true
@@ -619,6 +662,7 @@ endef
619662

620663
# Generate all docker-build targets
621664
$(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
665+
$(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
622666
$(eval $(call generate-docker-build-target,$(BACKEND_PIPER)))
623667
$(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE)))
624668
$(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE)))
@@ -663,7 +707,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
663707
docker-save-%: backend-images
664708
docker save local-ai-backend:$* -o backend-images/$*.tar
665709

666-
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
710+
docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
667711

668712
########################################################
669713
### Mock Backend for E2E Tests

0 commit comments

Comments
 (0)