Skip to content

Commit 5a2e0d5

Browse files
mudlerlocalai-bot
mudler
authored and
localai-bot
committed
feat(quantization): add quantization backend (mudler#9096)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 21012de commit 5a2e0d5

37 files changed

Lines changed: 2574 additions & 47 deletions

‎.github/workflows/backend.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,19 @@ jobs:
131131
dockerfile: "./backend/Dockerfile.python"
132132
context: "./"
133133
ubuntu-version: '2404'
134+
- build-type: ''
135+
cuda-major-version: ""
136+
cuda-minor-version: ""
137+
platforms: 'linux/amd64,linux/arm64'
138+
tag-latest: 'auto'
139+
tag-suffix: '-cpu-llama-cpp-quantization'
140+
runs-on: 'ubuntu-latest'
141+
base-image: "ubuntu:24.04"
142+
skip-drivers: 'true'
143+
backend: "llama-cpp-quantization"
144+
dockerfile: "./backend/Dockerfile.python"
145+
context: "./"
146+
ubuntu-version: '2404'
134147
- build-type: ''
135148
cuda-major-version: ""
136149
cuda-minor-version: ""
@@ -2412,6 +2425,9 @@ jobs:
24122425
tag-suffix: "-metal-darwin-arm64-local-store"
24132426
build-type: "metal"
24142427
lang: "go"
2428+
- backend: "llama-cpp-quantization"
2429+
tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization"
2430+
build-type: "mps"
24152431
with:
24162432
backend: ${{ matrix.backend }}
24172433
build-type: ${{ matrix.build-type }}

‎.github/workflows/test-extra.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,33 @@ jobs:
383383
run: |
384384
make --jobs=5 --output-sync=target -C backend/python/voxcpm
385385
make --jobs=5 --output-sync=target -C backend/python/voxcpm test
386+
tests-llama-cpp-quantization:
387+
runs-on: ubuntu-latest
388+
timeout-minutes: 30
389+
steps:
390+
- name: Clone
391+
uses: actions/checkout@v6
392+
with:
393+
submodules: true
394+
- name: Dependencies
395+
run: |
396+
sudo apt-get update
397+
sudo apt-get install -y build-essential cmake curl git python3-pip
398+
# Install UV
399+
curl -LsSf https://astral.sh/uv/install.sh | sh
400+
pip install --user --no-cache-dir grpcio-tools==1.64.1
401+
- name: Build llama-quantize from llama.cpp
402+
run: |
403+
git clone --depth 1 https://github.com/ggml-org/llama.cpp.git /tmp/llama.cpp
404+
cmake -B /tmp/llama.cpp/build -S /tmp/llama.cpp -DGGML_NATIVE=OFF
405+
cmake --build /tmp/llama.cpp/build --target llama-quantize -j$(nproc)
406+
sudo cp /tmp/llama.cpp/build/bin/llama-quantize /usr/local/bin/
407+
- name: Install backend
408+
run: |
409+
make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization
410+
- name: Test llama-cpp-quantization
411+
run: |
412+
make --jobs=5 --output-sync=target -C backend/python/llama-cpp-quantization test
386413
tests-acestep-cpp:
387414
runs-on: ubuntu-latest
388415
steps:

‎Makefile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Disable parallel execution for backend builds
2-
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl
2+
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization
33

44
GOCMD=go
55
GOTEST=$(GOCMD) test
@@ -575,6 +575,7 @@ BACKEND_WHISPERX = whisperx|python|.|false|true
575575
BACKEND_ACE_STEP = ace-step|python|.|false|true
576576
BACKEND_MLX_DISTRIBUTED = mlx-distributed|python|./|false|true
577577
BACKEND_TRL = trl|python|.|false|true
578+
BACKEND_LLAMA_CPP_QUANTIZATION = llama-cpp-quantization|python|.|false|true
578579

579580
# Helper function to build docker image for a backend
580581
# Usage: $(call docker-build-backend,BACKEND_NAME,DOCKERFILE_TYPE,BUILD_CONTEXT,PROGRESS_FLAG,NEEDS_BACKEND_ARG)
@@ -633,12 +634,13 @@ $(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP)))
633634
$(eval $(call generate-docker-build-target,$(BACKEND_ACESTEP_CPP)))
634635
$(eval $(call generate-docker-build-target,$(BACKEND_MLX_DISTRIBUTED)))
635636
$(eval $(call generate-docker-build-target,$(BACKEND_TRL)))
637+
$(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP_QUANTIZATION)))
636638

637639
# Pattern rule for docker-save targets
638640
docker-save-%: backend-images
639641
docker save local-ai-backend:$* -o backend-images/$*.tar
640642

641-
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl
643+
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization
642644

643645
########################################################
644646
### Mock Backend for E2E Tests

‎backend/backend.proto‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ service Backend {
4646
rpc StopFineTune(FineTuneStopRequest) returns (Result) {}
4747
rpc ListCheckpoints(ListCheckpointsRequest) returns (ListCheckpointsResponse) {}
4848
rpc ExportModel(ExportModelRequest) returns (Result) {}
49+
50+
// Quantization RPCs
51+
rpc StartQuantization(QuantizationRequest) returns (QuantizationJobResult) {}
52+
rpc QuantizationProgress(QuantizationProgressRequest) returns (stream QuantizationProgressUpdate) {}
53+
rpc StopQuantization(QuantizationStopRequest) returns (Result) {}
4954
}
5055

5156
// Define the empty request
@@ -637,3 +642,36 @@ message ExportModelRequest {
637642
string model = 5; // base model name (for merge operations)
638643
map<string, string> extra_options = 6;
639644
}
645+
646+
// Quantization messages
647+
648+
message QuantizationRequest {
649+
string model = 1; // HF model name or local path
650+
string quantization_type = 2; // q4_k_m, q5_k_m, q8_0, f16, etc.
651+
string output_dir = 3; // where to write output files
652+
string job_id = 4; // client-assigned job ID
653+
map<string, string> extra_options = 5; // hf_token, custom flags, etc.
654+
}
655+
656+
message QuantizationJobResult {
657+
string job_id = 1;
658+
bool success = 2;
659+
string message = 3;
660+
}
661+
662+
message QuantizationProgressRequest {
663+
string job_id = 1;
664+
}
665+
666+
message QuantizationProgressUpdate {
667+
string job_id = 1;
668+
float progress_percent = 2;
669+
string status = 3; // queued, downloading, converting, quantizing, completed, failed, stopped
670+
string message = 4;
671+
string output_file = 5; // set when completed — path to the output GGUF file
672+
map<string, float> extra_metrics = 6; // e.g. file_size_mb, compression_ratio
673+
}
674+
675+
message QuantizationStopRequest {
676+
string job_id = 1;
677+
}

‎backend/index.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3081,3 +3081,31 @@
30813081
uri: "quay.io/go-skynet/local-ai-backends:master-cublas-cuda13-trl"
30823082
mirrors:
30833083
- localai/localai-backends:master-cublas-cuda13-trl
3084+
## llama.cpp quantization backend
3085+
- &llama-cpp-quantization
3086+
name: "llama-cpp-quantization"
3087+
alias: "llama-cpp-quantization"
3088+
license: mit
3089+
icon: https://user-images.githubusercontent.com/1991296/230134379-7181e485-c521-4d23-a0d6-f7b3b61ba524.png
3090+
description: |
3091+
Model quantization backend using llama.cpp. Downloads HuggingFace models, converts them to GGUF format,
3092+
and quantizes them to various formats (q4_k_m, q5_k_m, q8_0, f16, etc.).
3093+
urls:
3094+
- https://github.com/ggml-org/llama.cpp
3095+
tags:
3096+
- quantization
3097+
- GGUF
3098+
- CPU
3099+
capabilities:
3100+
default: "cpu-llama-cpp-quantization"
3101+
metal: "metal-darwin-arm64-llama-cpp-quantization"
3102+
- !!merge <<: *llama-cpp-quantization
3103+
name: "cpu-llama-cpp-quantization"
3104+
uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-llama-cpp-quantization"
3105+
mirrors:
3106+
- localai/localai-backends:latest-cpu-llama-cpp-quantization
3107+
- !!merge <<: *llama-cpp-quantization
3108+
name: "metal-darwin-arm64-llama-cpp-quantization"
3109+
uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-llama-cpp-quantization"
3110+
mirrors:
3111+
- localai/localai-backends:latest-metal-darwin-arm64-llama-cpp-quantization

‎backend/python/llama-cpp-quantization/Makefile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Version of llama.cpp to fetch convert_hf_to_gguf.py from
2+
LLAMA_CPP_CONVERT_VERSION ?= master
3+
4+
.PHONY: llama-cpp-quantization
5+
llama-cpp-quantization:
6+
LLAMA_CPP_CONVERT_VERSION=$(LLAMA_CPP_CONVERT_VERSION) bash install.sh
7+
8+
.PHONY: run
9+
run: llama-cpp-quantization
10+
@echo "Running llama-cpp-quantization..."
11+
bash run.sh
12+
@echo "llama-cpp-quantization run."
13+
14+
.PHONY: test
15+
test: llama-cpp-quantization
16+
@echo "Testing llama-cpp-quantization..."
17+
bash test.sh
18+
@echo "llama-cpp-quantization tested."
19+
20+
.PHONY: protogen-clean
21+
protogen-clean:
22+
$(RM) backend_pb2_grpc.py backend_pb2.py
23+
24+
.PHONY: clean
25+
clean: protogen-clean
26+
rm -rf venv __pycache__

0 commit comments

Comments
 (0)