Skip to content

Commit 7a0e6ae

Browse files
mudlermudler
authored
feat(qwen3tts.cpp): add new backend (#9316)
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent e4bfc42 commit 7a0e6ae

17 files changed

Lines changed: 1168 additions & 4 deletions

File tree

‎.github/workflows/backend.yml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -613,6 +613,19 @@ jobs:
613613
dockerfile: "./backend/Dockerfile.golang"
614614
context: "./"
615615
ubuntu-version: '2404'
616+
- build-type: 'cublas'
617+
cuda-major-version: "12"
618+
cuda-minor-version: "8"
619+
platforms: 'linux/amd64'
620+
tag-latest: 'auto'
621+
tag-suffix: '-gpu-nvidia-cuda-12-qwen3-tts-cpp'
622+
runs-on: 'ubuntu-latest'
623+
base-image: "ubuntu:24.04"
624+
skip-drivers: 'false'
625+
backend: "qwen3-tts-cpp"
626+
dockerfile: "./backend/Dockerfile.golang"
627+
context: "./"
628+
ubuntu-version: '2404'
616629
- build-type: 'cublas'
617630
cuda-major-version: "12"
618631
cuda-minor-version: "8"
@@ -1225,6 +1238,19 @@ jobs:
12251238
dockerfile: "./backend/Dockerfile.golang"
12261239
context: "./"
12271240
ubuntu-version: '2404'
1241+
- build-type: 'cublas'
1242+
cuda-major-version: "13"
1243+
cuda-minor-version: "0"
1244+
platforms: 'linux/amd64'
1245+
tag-latest: 'auto'
1246+
tag-suffix: '-gpu-nvidia-cuda-13-qwen3-tts-cpp'
1247+
runs-on: 'ubuntu-latest'
1248+
base-image: "ubuntu:24.04"
1249+
skip-drivers: 'false'
1250+
backend: "qwen3-tts-cpp"
1251+
dockerfile: "./backend/Dockerfile.golang"
1252+
context: "./"
1253+
ubuntu-version: '2404'
12281254
- build-type: 'cublas'
12291255
cuda-major-version: "13"
12301256
cuda-minor-version: "0"
@@ -1238,6 +1264,19 @@ jobs:
12381264
backend: "acestep-cpp"
12391265
dockerfile: "./backend/Dockerfile.golang"
12401266
context: "./"
1267+
- build-type: 'cublas'
1268+
cuda-major-version: "13"
1269+
cuda-minor-version: "0"
1270+
platforms: 'linux/arm64'
1271+
skip-drivers: 'false'
1272+
tag-latest: 'auto'
1273+
tag-suffix: '-nvidia-l4t-cuda-13-arm64-qwen3-tts-cpp'
1274+
base-image: "ubuntu:24.04"
1275+
ubuntu-version: '2404'
1276+
runs-on: 'ubuntu-24.04-arm'
1277+
backend: "qwen3-tts-cpp"
1278+
dockerfile: "./backend/Dockerfile.golang"
1279+
context: "./"
12411280
- build-type: 'cublas'
12421281
cuda-major-version: "13"
12431282
cuda-minor-version: "0"
@@ -2222,6 +2261,85 @@ jobs:
22222261
dockerfile: "./backend/Dockerfile.golang"
22232262
context: "./"
22242263
ubuntu-version: '2404'
2264+
# qwen3-tts-cpp
2265+
- build-type: ''
2266+
cuda-major-version: ""
2267+
cuda-minor-version: ""
2268+
platforms: 'linux/amd64,linux/arm64'
2269+
tag-latest: 'auto'
2270+
tag-suffix: '-cpu-qwen3-tts-cpp'
2271+
runs-on: 'ubuntu-latest'
2272+
base-image: "ubuntu:24.04"
2273+
skip-drivers: 'false'
2274+
backend: "qwen3-tts-cpp"
2275+
dockerfile: "./backend/Dockerfile.golang"
2276+
context: "./"
2277+
ubuntu-version: '2404'
2278+
- build-type: 'sycl_f32'
2279+
cuda-major-version: ""
2280+
cuda-minor-version: ""
2281+
platforms: 'linux/amd64'
2282+
tag-latest: 'auto'
2283+
tag-suffix: '-gpu-intel-sycl-f32-qwen3-tts-cpp'
2284+
runs-on: 'ubuntu-latest'
2285+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
2286+
skip-drivers: 'false'
2287+
backend: "qwen3-tts-cpp"
2288+
dockerfile: "./backend/Dockerfile.golang"
2289+
context: "./"
2290+
ubuntu-version: '2404'
2291+
- build-type: 'sycl_f16'
2292+
cuda-major-version: ""
2293+
cuda-minor-version: ""
2294+
platforms: 'linux/amd64'
2295+
tag-latest: 'auto'
2296+
tag-suffix: '-gpu-intel-sycl-f16-qwen3-tts-cpp'
2297+
runs-on: 'ubuntu-latest'
2298+
base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04"
2299+
skip-drivers: 'false'
2300+
backend: "qwen3-tts-cpp"
2301+
dockerfile: "./backend/Dockerfile.golang"
2302+
context: "./"
2303+
ubuntu-version: '2404'
2304+
- build-type: 'vulkan'
2305+
cuda-major-version: ""
2306+
cuda-minor-version: ""
2307+
platforms: 'linux/amd64,linux/arm64'
2308+
tag-latest: 'auto'
2309+
tag-suffix: '-gpu-vulkan-qwen3-tts-cpp'
2310+
runs-on: 'ubuntu-latest'
2311+
base-image: "ubuntu:24.04"
2312+
skip-drivers: 'false'
2313+
backend: "qwen3-tts-cpp"
2314+
dockerfile: "./backend/Dockerfile.golang"
2315+
context: "./"
2316+
ubuntu-version: '2404'
2317+
- build-type: 'cublas'
2318+
cuda-major-version: "12"
2319+
cuda-minor-version: "0"
2320+
platforms: 'linux/arm64'
2321+
skip-drivers: 'false'
2322+
tag-latest: 'auto'
2323+
tag-suffix: '-nvidia-l4t-arm64-qwen3-tts-cpp'
2324+
base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0"
2325+
runs-on: 'ubuntu-24.04-arm'
2326+
backend: "qwen3-tts-cpp"
2327+
dockerfile: "./backend/Dockerfile.golang"
2328+
context: "./"
2329+
ubuntu-version: '2204'
2330+
- build-type: 'hipblas'
2331+
cuda-major-version: ""
2332+
cuda-minor-version: ""
2333+
platforms: 'linux/amd64'
2334+
tag-latest: 'auto'
2335+
tag-suffix: '-gpu-rocm-hipblas-qwen3-tts-cpp'
2336+
base-image: "rocm/dev-ubuntu-24.04:6.4.4"
2337+
runs-on: 'ubuntu-latest'
2338+
skip-drivers: 'false'
2339+
backend: "qwen3-tts-cpp"
2340+
dockerfile: "./backend/Dockerfile.golang"
2341+
context: "./"
2342+
ubuntu-version: '2404'
22252343
# voxtral
22262344
- build-type: ''
22272345
cuda-major-version: ""
@@ -2527,6 +2645,10 @@ jobs:
25272645
tag-suffix: "-metal-darwin-arm64-acestep-cpp"
25282646
build-type: "metal"
25292647
lang: "go"
2648+
- backend: "qwen3-tts-cpp"
2649+
tag-suffix: "-metal-darwin-arm64-qwen3-tts-cpp"
2650+
build-type: "metal"
2651+
lang: "go"
25302652
- backend: "voxtral"
25312653
tag-suffix: "-metal-darwin-arm64-voxtral"
25322654
build-type: "metal"

‎.github/workflows/bump_deps.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ jobs:
3838
variable: "SAM3_VERSION"
3939
branch: "main"
4040
file: "backend/go/sam3-cpp/Makefile"
41+
- repository: "predict-woo/qwen3-tts.cpp"
42+
variable: "QWEN3TTS_CPP_VERSION"
43+
branch: "main"
44+
file: "backend/go/qwen3-tts-cpp/Makefile"
4145
runs-on: ubuntu-latest
4246
steps:
4347
- uses: actions/checkout@v6

‎.github/workflows/test-extra.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ jobs:
3030
voxcpm: ${{ steps.detect.outputs.voxcpm }}
3131
llama-cpp-quantization: ${{ steps.detect.outputs.llama-cpp-quantization }}
3232
acestep-cpp: ${{ steps.detect.outputs.acestep-cpp }}
33+
qwen3-tts-cpp: ${{ steps.detect.outputs.qwen3-tts-cpp }}
3334
voxtral: ${{ steps.detect.outputs.voxtral }}
3435
kokoros: ${{ steps.detect.outputs.kokoros }}
3536
steps:
@@ -496,6 +497,38 @@ jobs:
496497
- name: Test acestep-cpp
497498
run: |
498499
make --jobs=5 --output-sync=target -C backend/go/acestep-cpp test
500+
tests-qwen3-tts-cpp:
501+
needs: detect-changes
502+
if: needs.detect-changes.outputs.qwen3-tts-cpp == 'true' || needs.detect-changes.outputs.run-all == 'true'
503+
runs-on: ubuntu-latest
504+
steps:
505+
- name: Clone
506+
uses: actions/checkout@v6
507+
with:
508+
submodules: true
509+
- name: Dependencies
510+
run: |
511+
sudo apt-get update
512+
sudo apt-get install -y build-essential cmake curl libopenblas-dev ffmpeg
513+
- name: Setup Go
514+
uses: actions/setup-go@v5
515+
- name: Display Go version
516+
run: go version
517+
- name: Proto Dependencies
518+
run: |
519+
# Install protoc
520+
curl -L -s https://github.com/protocolbuffers/protobuf/releases/download/v26.1/protoc-26.1-linux-x86_64.zip -o protoc.zip && \
521+
unzip -j -d /usr/local/bin protoc.zip bin/protoc && \
522+
rm protoc.zip
523+
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
524+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
525+
PATH="$PATH:$HOME/go/bin" make protogen-go
526+
- name: Build qwen3-tts-cpp
527+
run: |
528+
make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp
529+
- name: Test qwen3-tts-cpp
530+
run: |
531+
make --jobs=5 --output-sync=target -C backend/go/qwen3-tts-cpp test
499532
tests-voxtral:
500533
needs: detect-changes
501534
if: needs.detect-changes.outputs.voxtral == 'true' || needs.detect-changes.outputs.run-all == 'true'

‎Makefile‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Disable parallel execution for backend builds
2-
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp
2+
.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp
33

44
GOCMD=go
55
GOTEST=$(GOCMD) test
@@ -559,6 +559,7 @@ BACKEND_STABLEDIFFUSION_GGML = stablediffusion-ggml|golang|.|--progress=plain|tr
559559
BACKEND_WHISPER = whisper|golang|.|false|true
560560
BACKEND_VOXTRAL = voxtral|golang|.|false|true
561561
BACKEND_ACESTEP_CPP = acestep-cpp|golang|.|false|true
562+
BACKEND_QWEN3_TTS_CPP = qwen3-tts-cpp|golang|.|false|true
562563
BACKEND_OPUS = opus|golang|.|false|true
563564

564565
# Python backends with root context
@@ -651,6 +652,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM)))
651652
$(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX)))
652653
$(eval $(call generate-docker-build-target,$(BACKEND_ACE_STEP)))
653654
$(eval $(call generate-docker-build-target,$(BACKEND_ACESTEP_CPP)))
655+
$(eval $(call generate-docker-build-target,$(BACKEND_QWEN3_TTS_CPP)))
654656
$(eval $(call generate-docker-build-target,$(BACKEND_MLX_DISTRIBUTED)))
655657
$(eval $(call generate-docker-build-target,$(BACKEND_TRL)))
656658
$(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP_QUANTIZATION)))
@@ -661,7 +663,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SAM3_CPP)))
661663
docker-save-%: backend-images
662664
docker save local-ai-backend:$* -o backend-images/$*.tar
663665

664-
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp
666+
docker-build-backends: docker-build-llama-cpp docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-kokoros docker-build-sam3-cpp docker-build-qwen3-tts-cpp
665667

666668
########################################################
667669
### Mock Backend for E2E Tests

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
**LocalAI** is the open-source AI engine. Run any model - LLMs, vision, voice, image, video - on any hardware. No GPU required.
3333

3434
- **Drop-in API compatibility** — OpenAI, Anthropic, ElevenLabs APIs
35-
- **35+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
35+
- **36+ backends** — llama.cpp, vLLM, transformers, whisper, diffusers, MLX...
3636
- **Any hardware** — NVIDIA, AMD, Intel, Apple Silicon, Vulkan, or CPU-only
3737
- **Multi-user ready** — API key auth, user quotas, role-based access
3838
- **Built-in AI agents** — autonomous agents with tool use, RAG, MCP, and skills
@@ -185,7 +185,7 @@ For older news and full release notes, see [GitHub Releases](https://github.com/
185185

186186
## Supported Backends & Acceleration
187187

188-
LocalAI supports **35+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
188+
LocalAI supports **36+ backends** including llama.cpp, vLLM, transformers, whisper.cpp, diffusers, MLX, MLX-VLM, and many more. Hardware acceleration is available for **NVIDIA** (CUDA 12/13), **AMD** (ROCm), **Intel** (oneAPI/SYCL), **Apple Silicon** (Metal), **Vulkan**, and **NVIDIA Jetson** (L4T). All backends can be installed on-the-fly from the [Backend Gallery](https://localai.io/backends/).
189189

190190
See the full [Backend & Model Compatibility Table](https://localai.io/model-compatibility/) and [GPU Acceleration guide](https://localai.io/features/gpu-acceleration/).
191191

‎backend/go/qwen3-tts-cpp/CMakeLists.txt‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
cmake_minimum_required(VERSION 3.14)
2+
project(goqwen3ttscpp LANGUAGES C CXX)
3+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
4+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
5+
6+
set(QWEN3TTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/sources/qwen3-tts.cpp)
7+
8+
# Override upstream's CMAKE_CUDA_ARCHITECTURES before add_subdirectory.
9+
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
10+
set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real")
11+
endif()
12+
13+
# Build ggml from the upstream's submodule FIRST, so that ggml/ggml-base/ggml-cpu
14+
# CMake targets exist when the upstream project references them by name.
15+
# The upstream CMakeLists.txt uses target_link_libraries(... ggml ggml-base ggml-cpu)
16+
# with target_link_directories pointing at a pre-built ggml/build/. By adding ggml
17+
# as a subdirectory here, CMake resolves those names as targets instead.
18+
add_subdirectory(${QWEN3TTS_DIR}/ggml ggml EXCLUDE_FROM_ALL)
19+
20+
# Now add the upstream project
21+
add_subdirectory(${QWEN3TTS_DIR} qwen3tts EXCLUDE_FROM_ALL)
22+
23+
add_library(goqwen3ttscpp MODULE cpp/goqwen3ttscpp.cpp)
24+
target_link_libraries(goqwen3ttscpp PRIVATE qwen3_tts)
25+
26+
target_include_directories(goqwen3ttscpp PRIVATE ${QWEN3TTS_DIR}/src)
27+
target_include_directories(goqwen3ttscpp SYSTEM PRIVATE ${QWEN3TTS_DIR}/ggml/include)
28+
29+
# Link GPU backends if available
30+
foreach(backend blas cuda metal vulkan)
31+
if(TARGET ggml-${backend})
32+
target_link_libraries(goqwen3ttscpp PRIVATE ggml-${backend})
33+
string(TOUPPER ${backend} BACKEND_UPPER)
34+
target_compile_definitions(goqwen3ttscpp PRIVATE QWEN3TTS_HAVE_${BACKEND_UPPER})
35+
if(backend STREQUAL "cuda")
36+
find_package(CUDAToolkit QUIET)
37+
if(CUDAToolkit_FOUND)
38+
target_link_libraries(goqwen3ttscpp PRIVATE CUDA::cudart)
39+
endif()
40+
endif()
41+
endif()
42+
endforeach()
43+
44+
if(MSVC)
45+
target_compile_options(goqwen3ttscpp PRIVATE /W4 /wd4100 /wd4505)
46+
else()
47+
target_compile_options(goqwen3ttscpp PRIVATE -Wall -Wextra -Wshadow -Wconversion
48+
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
49+
endif()
50+
51+
if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
52+
target_link_libraries(goqwen3ttscpp PRIVATE stdc++fs)
53+
endif()
54+
55+
set_property(TARGET goqwen3ttscpp PROPERTY CXX_STANDARD 17)
56+
set_target_properties(goqwen3ttscpp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

0 commit comments

Comments
 (0)