diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index 5d4163f602fc..131787e0dd9c 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -326,6 +326,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "12" + cuda-minor-version: "8" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-12-funasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "12" cuda-minor-version: "8" @@ -1002,6 +1015,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'cublas' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-nvidia-cuda-13-funasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'cublas' cuda-major-version: "13" cuda-minor-version: "0" @@ -1337,6 +1363,19 @@ include: backend: "qwen-asr" dockerfile: "./backend/Dockerfile.python" context: "./" + - build-type: 'l4t' + cuda-major-version: "13" + cuda-minor-version: "0" + platforms: 'linux/arm64' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-cuda-13-arm64-funasr' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + ubuntu-version: '2404' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" - build-type: 'l4t' cuda-major-version: "13" cuda-minor-version: "0" @@ -2212,6 +2251,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-funasr' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'hipblas' cuda-major-version: "" cuda-minor-version: "" @@ -2492,6 +2544,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2204' + - build-type: 'l4t' + cuda-major-version: "12" + cuda-minor-version: "0" + platforms: 'linux/arm64' + tag-latest: 'auto' + tag-suffix: '-nvidia-l4t-funasr' + runs-on: 'ubuntu-24.04-arm' + base-image: "nvcr.io/nvidia/l4t-jetpack:r36.4.0" + skip-drivers: 'true' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2204' - build-type: 'l4t' cuda-major-version: "12" cuda-minor-version: "0" @@ -2701,6 +2766,19 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: 'intel' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-intel-funasr' + runs-on: 'ubuntu-latest' + base-image: "intel/oneapi-basekit:2025.3.0-0-devel-ubuntu24.04" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: 'intel' cuda-major-version: "" cuda-minor-version: "" @@ -5462,6 +5540,20 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-funasr' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -5476,6 +5568,20 @@ include: dockerfile: "./backend/Dockerfile.python" context: "./" ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-funasr' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "funasr" + dockerfile: "./backend/Dockerfile.python" + context: "./" + ubuntu-version: '2404' - build-type: '' cuda-major-version: "" cuda-minor-version: "" @@ -5823,6 +5929,9 @@ includeDarwin: - backend: "qwen-asr" tag-suffix: "-metal-darwin-arm64-qwen-asr" build-type: "mps" + - backend: "funasr" + tag-suffix: "-metal-darwin-arm64-funasr" + build-type: "mps" - backend: "nemo" tag-suffix: "-metal-darwin-arm64-nemo" build-type: "mps" diff --git a/Makefile b/Makefile index 16881ec22cae..de0ab6ba079a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/funasr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin GOCMD=go GOTEST=$(GOCMD) test @@ -594,6 +594,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/python/fish-speech $(MAKE) -C backend/python/faster-qwen3-tts $(MAKE) -C backend/python/qwen-asr + $(MAKE) -C backend/python/funasr $(MAKE) -C backend/python/nemo $(MAKE) -C backend/python/voxcpm $(MAKE) -C backend/python/faster-whisper @@ -623,6 +624,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/python/fish-speech test $(MAKE) -C backend/python/faster-qwen3-tts test $(MAKE) -C backend/python/qwen-asr test + $(MAKE) -C backend/python/funasr test $(MAKE) -C backend/python/nemo test $(MAKE) -C backend/python/voxcpm test $(MAKE) -C backend/python/faster-whisper test @@ -1302,6 +1304,7 @@ BACKEND_QWEN_TTS = qwen-tts|python|.|false|true BACKEND_FISH_SPEECH = fish-speech|python|.|false|true BACKEND_FASTER_QWEN3_TTS = faster-qwen3-tts|python|.|false|true BACKEND_QWEN_ASR = qwen-asr|python|.|false|true +BACKEND_FUNASR = funasr|python|.|false|true BACKEND_NEMO = nemo|python|.|false|true BACKEND_VOXCPM = voxcpm|python|.|false|true BACKEND_WHISPERX = whisperx|python|.|false|true @@ -1389,6 +1392,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_TTS))) $(eval $(call generate-docker-build-target,$(BACKEND_FISH_SPEECH))) $(eval $(call generate-docker-build-target,$(BACKEND_FASTER_QWEN3_TTS))) $(eval $(call generate-docker-build-target,$(BACKEND_QWEN_ASR))) +$(eval $(call generate-docker-build-target,$(BACKEND_FUNASR))) $(eval $(call generate-docker-build-target,$(BACKEND_NEMO))) $(eval $(call generate-docker-build-target,$(BACKEND_VOXCPM))) $(eval $(call generate-docker-build-target,$(BACKEND_WHISPERX))) @@ -1415,7 +1419,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-funasr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter ######################################################## ### Mock Backend for E2E Tests diff --git a/backend/README.md b/backend/README.md index 0e92a0f03c58..b72dd01b6e96 100644 --- a/backend/README.md +++ b/backend/README.md @@ -47,7 +47,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **mlx**: Apple Silicon optimization - **diffusers**: Stable Diffusion models - **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation -- **Audio**: coqui, faster-whisper, kitten-tts +- **Audio**: coqui, faster-whisper, funasr, kitten-tts - **Vision**: mlx-vlm, rfdetr - **Specialized**: rerankers, chatterbox, kokoro diff --git a/backend/index.yaml b/backend/index.yaml index 5d6eb8d50f08..2164f756e1a6 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -1585,6 +1585,32 @@ nvidia-l4t-cuda-12: "nvidia-l4t-faster-qwen3-tts" nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-faster-qwen3-tts" icon: https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png +- &funasr + urls: + - https://github.com/modelscope/FunASR + description: | + FunASR is an industrial-grade speech recognition toolkit supporting 50+ languages. + Includes SenseVoice (170x realtime, emotion detection), Paraformer (highest Chinese + accuracy), and built-in VAD, punctuation restoration, and speaker diarization. + tags: + - speech-recognition + - ASR + - multilingual + license: mit + name: "funasr" + alias: "funasr" + capabilities: + nvidia: "cuda12-funasr" + intel: "intel-funasr" + amd: "rocm-funasr" + metal: "metal-funasr" + default: "cpu-funasr" + nvidia-cuda-13: "cuda13-funasr" + nvidia-cuda-12: "cuda12-funasr" + nvidia-l4t: "nvidia-l4t-funasr" + nvidia-l4t-cuda-12: "nvidia-l4t-funasr" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr" + icon: https://avatars.githubusercontent.com/u/109454077 - &qwen-asr urls: - https://github.com/QwenLM/Qwen3-ASR @@ -6430,3 +6456,97 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-supertonic" mirrors: - localai/localai-backends:master-metal-darwin-arm64-supertonic +## funasr +- !!merge <<: *funasr + name: "funasr-development" + capabilities: + nvidia: "cuda12-funasr-development" + intel: "intel-funasr-development" + amd: "rocm-funasr-development" + nvidia-l4t: "nvidia-l4t-funasr-development" + metal: "metal-funasr-development" + default: "cpu-funasr-development" + nvidia-cuda-13: "cuda13-funasr-development" + nvidia-cuda-12: "cuda12-funasr-development" + nvidia-l4t-cuda-12: "nvidia-l4t-funasr-development" + nvidia-l4t-cuda-13: "cuda13-nvidia-l4t-arm64-funasr-development" +- !!merge <<: *funasr + name: "cpu-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-funasr" + mirrors: + - localai/localai-backends:latest-cpu-funasr +- !!merge <<: *funasr + name: "cpu-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-funasr" + mirrors: + - localai/localai-backends:master-cpu-funasr +- !!merge <<: *funasr + name: "cuda12-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-funasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-12-funasr +- !!merge <<: *funasr + name: "cuda12-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-funasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-12-funasr +- !!merge <<: *funasr + name: "cuda13-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-13-funasr" + mirrors: + - localai/localai-backends:latest-gpu-nvidia-cuda-13-funasr +- !!merge <<: *funasr + name: "cuda13-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-13-funasr" + mirrors: + - localai/localai-backends:master-gpu-nvidia-cuda-13-funasr +- !!merge <<: *funasr + name: "intel-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-funasr" + mirrors: + - localai/localai-backends:latest-gpu-intel-funasr +- !!merge <<: *funasr + name: "intel-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-funasr" + mirrors: + - localai/localai-backends:master-gpu-intel-funasr +- !!merge <<: *funasr + name: "rocm-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-funasr" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-funasr +- !!merge <<: *funasr + name: "rocm-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-funasr" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-funasr +- !!merge <<: *funasr + name: "nvidia-l4t-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-funasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-funasr +- !!merge <<: *funasr + name: "nvidia-l4t-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-funasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-funasr +- !!merge <<: *funasr + name: "cuda13-nvidia-l4t-arm64-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr" + mirrors: + - localai/localai-backends:latest-nvidia-l4t-cuda-13-arm64-funasr +- !!merge <<: *funasr + name: "cuda13-nvidia-l4t-arm64-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-nvidia-l4t-cuda-13-arm64-funasr" + mirrors: + - localai/localai-backends:master-nvidia-l4t-cuda-13-arm64-funasr +- !!merge <<: *funasr + name: "metal-funasr" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-funasr" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-funasr +- !!merge <<: *funasr + name: "metal-funasr-development" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-funasr diff --git a/backend/python/README.md b/backend/python/README.md index 1864502f5925..342d19bc2cfd 100644 --- a/backend/python/README.md +++ b/backend/python/README.md @@ -20,6 +20,7 @@ The Python backends use a unified build system based on `libbackend.sh` that pro ### Audio & Speech - **coqui** - Coqui TTS models - **faster-whisper** - Fast Whisper speech recognition +- **funasr** - Local multilingual transcription with FunASR and SenseVoice - **kitten-tts** - Lightweight TTS - **mlx-audio** - Apple Silicon audio processing - **chatterbox** - TTS model diff --git a/backend/python/funasr/Makefile b/backend/python/funasr/Makefile new file mode 100644 index 000000000000..1bb15fe3af55 --- /dev/null +++ b/backend/python/funasr/Makefile @@ -0,0 +1,23 @@ +.PHONY: funasr +funasr: + bash install.sh + +.PHONY: run +run: funasr + @echo "Running funasr..." + bash run.sh + @echo "funasr run." + +.PHONY: test +test: funasr + @echo "Testing funasr..." + bash test.sh + @echo "funasr tested." + +.PHONY: protogen-clean +protogen-clean: + $(RM) backend_pb2_grpc.py backend_pb2.py + +.PHONY: clean +clean: protogen-clean + rm -rf venv __pycache__ diff --git a/backend/python/funasr/backend.py b/backend/python/funasr/backend.py new file mode 100644 index 000000000000..dcaf80ffcfd3 --- /dev/null +++ b/backend/python/funasr/backend.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +gRPC backend for LocalAI wrapping FunASR (SenseVoice / Paraformer). +""" +from concurrent import futures +import time +import argparse +import signal +import sys +import os +import backend_pb2 +import backend_pb2_grpc +import torch + +import grpc +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'common')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'common')) +from grpc_auth import get_auth_interceptors + + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 +MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1')) + + +class BackendServicer(backend_pb2_grpc.BackendServicer): + def Health(self, request, context): + return backend_pb2.Reply(message=bytes("OK", 'utf-8')) + + def LoadModel(self, request, context): + from funasr import AutoModel + + device = "cpu" + if request.CUDA and torch.cuda.is_available(): + device = "cuda" + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + device = "xpu" + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + device = "mps" + + model_id = request.Model or "iic/SenseVoiceSmall" + candidate_devices = [device] + if device in ("xpu", "mps"): + candidate_devices.append("cpu") + + for candidate_device in candidate_devices: + try: + print( + f"Loading FunASR model: {model_id} on {candidate_device}", + file=sys.stderr, + ) + self.model = AutoModel( + model=model_id, + vad_model="fsmn-vad", + device=candidate_device, + disable_update=True, + ) + print("FunASR model loaded successfully", file=sys.stderr) + break + except Exception as err: + if candidate_device != candidate_devices[-1]: + print( + f"[WARN] FunASR {candidate_device} initialization failed: " + f"{err}; retrying on cpu", + file=sys.stderr, + ) + continue + print(f"[ERROR] LoadModel failed: {err}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return backend_pb2.Result(success=False, message=str(err)) + + return backend_pb2.Result(message="Model loaded successfully", success=True) + + def AudioTranscription(self, request, context): + from funasr.utils.postprocess_utils import rich_transcription_postprocess + + result_segments = [] + text = "" + try: + audio_path = request.dst + if not audio_path or not os.path.exists(audio_path): + print(f"Error: Audio file not found: {audio_path}", file=sys.stderr) + return backend_pb2.TranscriptResult(segments=[], text="") + + language = None + if request.language and request.language.strip(): + language = request.language.strip() + + kwargs = {} + if language: + kwargs["language"] = language + + results = self.model.generate(input=audio_path, **kwargs) + + if not results: + return backend_pb2.TranscriptResult(segments=[], text="") + + for idx, r in enumerate(results): + seg_text = r.get("text", "") if isinstance(r, dict) else str(r) + seg_text = rich_transcription_postprocess(seg_text) + text += seg_text + result_segments.append(backend_pb2.TranscriptSegment( + id=idx, + start=0, + end=0, + text=seg_text, + )) + + except Exception as err: + print(f"Error in AudioTranscription: {err}", file=sys.stderr) + import traceback + traceback.print_exc(file=sys.stderr) + return backend_pb2.TranscriptResult(segments=[], text="") + + return backend_pb2.TranscriptResult(segments=result_segments, text=text) + + +def serve(address): + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=MAX_WORKERS), + options=[ + ('grpc.max_message_length', 50 * 1024 * 1024), + ('grpc.max_send_message_length', 50 * 1024 * 1024), + ('grpc.max_receive_message_length', 50 * 1024 * 1024), + ], + interceptors=get_auth_interceptors(), + ) + backend_pb2_grpc.add_BackendServicer_to_server(BackendServicer(), server) + server.add_insecure_port(address) + server.start() + print("Server started. Listening on: " + address, file=sys.stderr) + + def signal_handler(sig, frame): + print("Received termination signal. Shutting down...") + server.stop(0) + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the gRPC server.") + parser.add_argument("--addr", default="localhost:50051", help="The address to bind the server to.") + args = parser.parse_args() + serve(args.addr) diff --git a/backend/python/funasr/install.sh b/backend/python/funasr/install.sh new file mode 100755 index 000000000000..71c9e79a93f9 --- /dev/null +++ b/backend/python/funasr/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +EXTRA_PIP_INSTALL_FLAGS="--no-build-isolation" + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +if [ "x${BUILD_PROFILE}" == "xintel" ]; then + EXTRA_PIP_INSTALL_FLAGS+=" --upgrade --index-strategy=unsafe-first-match" +fi + +PYTHON_VERSION="3.12" +PYTHON_PATCH="12" +PY_STANDALONE_TAG="20251120" + +installRequirements diff --git a/backend/python/funasr/requirements-cpu.txt b/backend/python/funasr/requirements-cpu.txt new file mode 100644 index 000000000000..112799ff99f3 --- /dev/null +++ b/backend/python/funasr/requirements-cpu.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cpu +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-cublas12.txt b/backend/python/funasr/requirements-cublas12.txt new file mode 100644 index 000000000000..54ec05fdfba3 --- /dev/null +++ b/backend/python/funasr/requirements-cublas12.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cu121 +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-cublas13.txt b/backend/python/funasr/requirements-cublas13.txt new file mode 100644 index 000000000000..5ae56f880acb --- /dev/null +++ b/backend/python/funasr/requirements-cublas13.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cu130 +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-hipblas.txt b/backend/python/funasr/requirements-hipblas.txt new file mode 100644 index 000000000000..b050a7043b06 --- /dev/null +++ b/backend/python/funasr/requirements-hipblas.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/rocm7.0 +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-intel.txt b/backend/python/funasr/requirements-intel.txt new file mode 100644 index 000000000000..f00e3e925769 --- /dev/null +++ b/backend/python/funasr/requirements-intel.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/xpu +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-l4t12.txt b/backend/python/funasr/requirements-l4t12.txt new file mode 100644 index 000000000000..2b55b8d11443 --- /dev/null +++ b/backend/python/funasr/requirements-l4t12.txt @@ -0,0 +1,4 @@ +--extra-index-url https://pypi.jetson-ai-lab.io/jp6/cu129/ +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-l4t13.txt b/backend/python/funasr/requirements-l4t13.txt new file mode 100644 index 000000000000..5ae56f880acb --- /dev/null +++ b/backend/python/funasr/requirements-l4t13.txt @@ -0,0 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cu130 +torch +torchaudio +funasr diff --git a/backend/python/funasr/requirements-mps.txt b/backend/python/funasr/requirements-mps.txt new file mode 100644 index 000000000000..c6a89dde7840 --- /dev/null +++ b/backend/python/funasr/requirements-mps.txt @@ -0,0 +1,3 @@ +torch==2.7.1 +torchaudio==2.7.1 +funasr diff --git a/backend/python/funasr/requirements.txt b/backend/python/funasr/requirements.txt new file mode 100644 index 000000000000..9ce0da738095 --- /dev/null +++ b/backend/python/funasr/requirements.txt @@ -0,0 +1,5 @@ +grpcio==1.71.0 +protobuf +certifi +packaging==24.1 +setuptools diff --git a/backend/python/funasr/run.sh b/backend/python/funasr/run.sh new file mode 100755 index 000000000000..eae121f37b0b --- /dev/null +++ b/backend/python/funasr/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +startBackend $@ diff --git a/backend/python/funasr/test.py b/backend/python/funasr/test.py new file mode 100644 index 000000000000..9e890001f6aa --- /dev/null +++ b/backend/python/funasr/test.py @@ -0,0 +1,295 @@ +import importlib +import os +import re +import sys +import types +import unittest + + +class _Reply: + def __init__(self, message=b""): + self.message = message + + +class _Result: + def __init__(self, message="", success=False): + self.message = message + self.success = success + + +class _TranscriptSegment: + def __init__(self, id=0, start=0, end=0, text=""): + self.id = id + self.start = start + self.end = end + self.text = text + + +class _TranscriptResult: + def __init__(self, segments=None, text=""): + self.segments = segments or [] + self.text = text + + +class _FakeBackendServicer: + pass + + +class _FakeTorch: + cuda = types.SimpleNamespace(is_available=lambda: False) + xpu = types.SimpleNamespace(is_available=lambda: False) + backends = types.SimpleNamespace(mps=types.SimpleNamespace(is_available=lambda: False)) + + +class _FakeAutoModel: + instances = [] + attempts = [] + fail_devices = set() + + def __init__(self, **kwargs): + _FakeAutoModel.attempts.append(kwargs) + if kwargs.get("device") in _FakeAutoModel.fail_devices: + raise RuntimeError(f"unsupported device: {kwargs['device']}") + self.kwargs = kwargs + self.generate_calls = [] + self.results = [{"text": "hello"}, {"text": " world"}] + _FakeAutoModel.instances.append(self) + + def generate(self, **kwargs): + self.generate_calls.append(kwargs) + return self.results + + +def _install_stubs(): + sys.modules["backend_pb2"] = types.SimpleNamespace( + Reply=_Reply, + Result=_Result, + TranscriptSegment=_TranscriptSegment, + TranscriptResult=_TranscriptResult, + ) + sys.modules["backend_pb2_grpc"] = types.SimpleNamespace( + BackendServicer=_FakeBackendServicer, + add_BackendServicer_to_server=lambda *args, **kwargs: None, + ) + sys.modules["grpc"] = types.SimpleNamespace( + RpcMethodHandler=object, + ServerInterceptor=object, + StatusCode=types.SimpleNamespace(UNAUTHENTICATED="UNAUTHENTICATED"), + aio=types.SimpleNamespace(ServerInterceptor=object), + server=lambda *args, **kwargs: None, + ) + sys.modules["torch"] = _FakeTorch + funasr = types.ModuleType("funasr") + funasr.__path__ = [] + funasr.AutoModel = _FakeAutoModel + funasr_utils = types.ModuleType("funasr.utils") + funasr_utils.__path__ = [] + postprocess_utils = types.ModuleType("funasr.utils.postprocess_utils") + postprocess_utils.rich_transcription_postprocess = lambda text: re.sub( + r"<\|.*?\|>", "", text + ) + sys.modules["funasr"] = funasr + sys.modules["funasr.utils"] = funasr_utils + sys.modules["funasr.utils.postprocess_utils"] = postprocess_utils + + +def _load_backend(): + _install_stubs() + sys.modules.pop("backend", None) + _FakeAutoModel.instances.clear() + _FakeAutoModel.attempts.clear() + _FakeAutoModel.fail_devices.clear() + return importlib.import_module("backend") + + +class TestFunASRBackend(unittest.TestCase): + def test_torch_profiles_install_torchaudio(self): + backend_dir = os.path.dirname(__file__) + + for profile in ( + "cpu", + "cublas12", + "cublas13", + "hipblas", + "intel", + "l4t12", + "l4t13", + "mps", + ): + with self.subTest(profile=profile): + requirements_path = os.path.join( + backend_dir, f"requirements-{profile}.txt" + ) + with open(requirements_path, encoding="utf-8") as requirements_file: + requirements = { + line.strip().split("=", 1)[0] + for line in requirements_file + if line.strip() and not line.lstrip().startswith(("#", "--")) + } + + self.assertIn("torchaudio", requirements) + + def test_accelerator_profiles_use_current_pytorch_indexes(self): + backend_dir = os.path.dirname(__file__) + expected_indexes = { + "cublas13": "https://download.pytorch.org/whl/cu130", + "hipblas": "https://download.pytorch.org/whl/rocm7.0", + "intel": "https://download.pytorch.org/whl/xpu", + "l4t12": "https://pypi.jetson-ai-lab.io/jp6/cu129/", + "l4t13": "https://download.pytorch.org/whl/cu130", + } + + for profile, expected_index in expected_indexes.items(): + with self.subTest(profile=profile): + requirements_path = os.path.join( + backend_dir, f"requirements-{profile}.txt" + ) + with open(requirements_path, encoding="utf-8") as requirements_file: + requirements = requirements_file.read() + + self.assertIn( + f"--extra-index-url {expected_index}\n", + requirements, + ) + + def test_health_returns_ok(self): + backend = _load_backend() + servicer = backend.BackendServicer() + + reply = servicer.Health(types.SimpleNamespace(), None) + + self.assertEqual(reply.message, b"OK") + + def test_load_model_uses_default_sensevoice_model_on_cpu(self): + backend = _load_backend() + servicer = backend.BackendServicer() + + result = servicer.LoadModel(types.SimpleNamespace(Model="", CUDA=False), None) + + self.assertTrue(result.success, result.message) + self.assertEqual(result.message, "Model loaded successfully") + self.assertEqual(_FakeAutoModel.instances[0].kwargs["model"], "iic/SenseVoiceSmall") + self.assertEqual(_FakeAutoModel.instances[0].kwargs["vad_model"], "fsmn-vad") + self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu") + self.assertTrue(_FakeAutoModel.instances[0].kwargs["disable_update"]) + + def test_load_model_uses_xpu_when_available(self): + backend = _load_backend() + servicer = backend.BackendServicer() + original_xpu = _FakeTorch.xpu + _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True) + self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu) + + result = servicer.LoadModel( + types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None + ) + + self.assertTrue(result.success, result.message) + self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "xpu") + + def test_load_model_prefers_requested_cuda_over_xpu_and_mps(self): + backend = _load_backend() + servicer = backend.BackendServicer() + original_cuda = _FakeTorch.cuda + original_xpu = _FakeTorch.xpu + original_mps = _FakeTorch.backends.mps + _FakeTorch.cuda = types.SimpleNamespace(is_available=lambda: True) + _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True) + _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True) + self.addCleanup(setattr, _FakeTorch, "cuda", original_cuda) + self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu) + self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps) + + result = servicer.LoadModel( + types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=True), None + ) + + self.assertTrue(result.success, result.message) + self.assertEqual( + [attempt["device"] for attempt in _FakeAutoModel.attempts], ["cuda"] + ) + + def test_load_model_retries_cpu_when_xpu_initialization_fails(self): + backend = _load_backend() + servicer = backend.BackendServicer() + original_xpu = _FakeTorch.xpu + _FakeTorch.xpu = types.SimpleNamespace(is_available=lambda: True) + _FakeAutoModel.fail_devices.add("xpu") + self.addCleanup(setattr, _FakeTorch, "xpu", original_xpu) + + result = servicer.LoadModel( + types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None + ) + + self.assertTrue(result.success, result.message) + self.assertEqual( + [attempt["device"] for attempt in _FakeAutoModel.attempts], + ["xpu", "cpu"], + ) + self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu") + + def test_load_model_retries_cpu_when_mps_initialization_fails(self): + backend = _load_backend() + servicer = backend.BackendServicer() + original_mps = _FakeTorch.backends.mps + _FakeTorch.backends.mps = types.SimpleNamespace(is_available=lambda: True) + _FakeAutoModel.fail_devices.add("mps") + self.addCleanup(setattr, _FakeTorch.backends, "mps", original_mps) + + result = servicer.LoadModel( + types.SimpleNamespace(Model="iic/SenseVoiceSmall", CUDA=False), None + ) + + self.assertTrue(result.success, result.message) + self.assertEqual( + [attempt["device"] for attempt in _FakeAutoModel.attempts], + ["mps", "cpu"], + ) + self.assertEqual(_FakeAutoModel.instances[0].kwargs["device"], "cpu") + + def test_audio_transcription_passes_language_and_builds_segments(self): + backend = _load_backend() + servicer = backend.BackendServicer() + servicer.model = _FakeAutoModel() + audio_path = os.path.abspath(__file__) + request = types.SimpleNamespace(dst=audio_path, language=" zh ") + + result = servicer.AudioTranscription(request, None) + + self.assertEqual(result.text, "hello world") + self.assertEqual([segment.text for segment in result.segments], ["hello", " world"]) + self.assertEqual(servicer.model.generate_calls, [{"input": audio_path, "language": "zh"}]) + + def test_audio_transcription_cleans_sensevoice_tags(self): + backend = _load_backend() + servicer = backend.BackendServicer() + servicer.model = _FakeAutoModel() + servicer.model.results = [ + {"text": "<|zh|><|NEUTRAL|><|Speech|><|woitn|>hello"} + ] + + result = servicer.AudioTranscription( + types.SimpleNamespace(dst=os.path.abspath(__file__), language=""), + None, + ) + + self.assertEqual(result.text, "hello") + self.assertEqual([segment.text for segment in result.segments], ["hello"]) + + def test_audio_transcription_missing_file_returns_empty_result(self): + backend = _load_backend() + servicer = backend.BackendServicer() + servicer.model = _FakeAutoModel() + + result = servicer.AudioTranscription( + types.SimpleNamespace(dst="/tmp/localai-funasr-missing.wav", language=""), + None, + ) + + self.assertEqual(result.text, "") + self.assertEqual(result.segments, []) + self.assertEqual(servicer.model.generate_calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/python/funasr/test.sh b/backend/python/funasr/test.sh new file mode 100755 index 000000000000..a3fc736a017f --- /dev/null +++ b/backend/python/funasr/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +backend_dir=$(dirname $0) +if [ -d $backend_dir/common ]; then + source $backend_dir/common/libbackend.sh +else + source $backend_dir/../common/libbackend.sh +fi + +runUnittests "$@" diff --git a/core/gallery/importers/funasr.go b/core/gallery/importers/funasr.go new file mode 100644 index 000000000000..a27135fcc3b0 --- /dev/null +++ b/core/gallery/importers/funasr.go @@ -0,0 +1,99 @@ +package importers + +import ( + "encoding/json" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/schema" + "go.yaml.in/yaml/v2" +) + +const ( + funASRSenseVoiceHF = "FunAudioLLM/SenseVoiceSmall" + funASRSenseVoiceModelScope = "iic/SenseVoiceSmall" +) + +var _ Importer = &FunASRImporter{} + +// FunASRImporter auto-detects only the official SenseVoiceSmall repository. +// Other FunAudioLLM and iic models require preferences.backend="funasr". +type FunASRImporter struct{} + +func (i *FunASRImporter) Name() string { return "funasr" } +func (i *FunASRImporter) Modality() string { return "asr" } +func (i *FunASRImporter) AutoDetects() bool { return true } + +func (i *FunASRImporter) Match(details Details) bool { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return false + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return false + } + } + + if backend, ok := preferencesMap["backend"].(string); ok { + return backend == "funasr" + } + + return details.HuggingFace != nil && + strings.EqualFold(details.HuggingFace.ModelID, funASRSenseVoiceHF) +} + +func (i *FunASRImporter) Import(details Details) (gallery.ModelConfig, error) { + preferences, err := details.Preferences.MarshalJSON() + if err != nil { + return gallery.ModelConfig{}, err + } + preferencesMap := make(map[string]any) + if len(preferences) > 0 { + if err := json.Unmarshal(preferences, &preferencesMap); err != nil { + return gallery.ModelConfig{}, err + } + } + + name, ok := preferencesMap["name"].(string) + if !ok { + name = filepath.Base(details.URI) + } + + description, ok := preferencesMap["description"].(string) + if !ok { + description = "Imported from " + details.URI + } + + model := details.URI + if details.HuggingFace != nil && details.HuggingFace.ModelID != "" { + model = details.HuggingFace.ModelID + } + if strings.EqualFold(model, funASRSenseVoiceHF) { + model = funASRSenseVoiceModelScope + } + + modelConfig := config.ModelConfig{ + Name: name, + Description: description, + Backend: "funasr", + KnownUsecaseStrings: []string{"transcript"}, + PredictionOptions: schema.PredictionOptions{ + BasicModelRequest: schema.BasicModelRequest{Model: model}, + }, + } + + data, err := yaml.Marshal(modelConfig) + if err != nil { + return gallery.ModelConfig{}, err + } + + return gallery.ModelConfig{ + Name: name, + Description: description, + ConfigFile: string(data), + }, nil +} diff --git a/core/gallery/importers/funasr_test.go b/core/gallery/importers/funasr_test.go new file mode 100644 index 000000000000..9a60a94654d5 --- /dev/null +++ b/core/gallery/importers/funasr_test.go @@ -0,0 +1,138 @@ +package importers_test + +import ( + "encoding/json" + "fmt" + + "github.com/mudler/LocalAI/core/gallery/importers" + hfapi "github.com/mudler/LocalAI/pkg/huggingface-api" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func registeredImporter(name string) importers.Importer { + for _, importer := range importers.Registry() { + if importer.Name() == name { + return importer + } + } + return nil +} + +var _ = Describe("FunASRImporter", func() { + Context("registry metadata", func() { + It("registers an auto-detected ASR backend", func() { + importer := registeredImporter("funasr") + + Expect(importer).ToNot(BeNil()) + if importer == nil { + return + } + Expect(importer.Modality()).To(Equal("asr")) + Expect(importer.AutoDetects()).To(BeTrue()) + }) + }) + + Context("matching", func() { + DescribeTable("matches only the official SenseVoiceSmall repository or an explicit preference", + func(details importers.Details, expected bool) { + importer := registeredImporter("funasr") + Expect(importer).ToNot(BeNil()) + if importer == nil { + return + } + Expect(importer.Match(details)).To(Equal(expected)) + }, + Entry("official FunAudioLLM/SenseVoiceSmall", + importers.Details{ + URI: "https://huggingface.co/FunAudioLLM/SenseVoiceSmall", + HuggingFace: &hfapi.ModelDetails{ + Author: "FunAudioLLM", + ModelID: "FunAudioLLM/SenseVoiceSmall", + }, + }, + true, + ), + Entry("same owner but a different model", + importers.Details{ + URI: "https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512", + HuggingFace: &hfapi.ModelDetails{ + Author: "FunAudioLLM", + ModelID: "FunAudioLLM/Fun-ASR-Nano-2512", + }, + }, + false, + ), + Entry("same model name under another owner", + importers.Details{ + URI: "https://huggingface.co/community/SenseVoiceSmall", + HuggingFace: &hfapi.ModelDetails{ + Author: "community", + ModelID: "community/SenseVoiceSmall", + }, + }, + false, + ), + Entry("explicit funasr preference for an arbitrary model", + importers.Details{ + URI: "iic/paraformer-zh", + Preferences: json.RawMessage(`{"backend":"funasr"}`), + }, + true, + ), + Entry("another explicit backend prevents auto-detection", + importers.Details{ + URI: "https://huggingface.co/FunAudioLLM/SenseVoiceSmall", + Preferences: json.RawMessage(`{"backend":"qwen-asr"}`), + HuggingFace: &hfapi.ModelDetails{ + Author: "FunAudioLLM", + ModelID: "FunAudioLLM/SenseVoiceSmall", + }, + }, + false, + ), + ) + }) + + Context("import", func() { + It("maps the official Hugging Face model to the working ModelScope identifier", func() { + importer := registeredImporter("funasr") + Expect(importer).ToNot(BeNil()) + if importer == nil { + return + } + details := importers.Details{ + URI: "https://huggingface.co/FunAudioLLM/SenseVoiceSmall", + HuggingFace: &hfapi.ModelDetails{ + Author: "FunAudioLLM", + ModelID: "FunAudioLLM/SenseVoiceSmall", + }, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: funasr")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: iic/SenseVoiceSmall")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- transcript")) + }) + + It("preserves an explicitly selected model identifier", func() { + importer := registeredImporter("funasr") + Expect(importer).ToNot(BeNil()) + if importer == nil { + return + } + details := importers.Details{ + URI: "iic/paraformer-zh", + Preferences: json.RawMessage(`{"backend":"funasr"}`), + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Error: %v", err)) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: funasr")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("model: iic/paraformer-zh")) + }) + }) +}) diff --git a/core/gallery/importers/importers.go b/core/gallery/importers/importers.go index 8c156144d44c..ce7627d3c39a 100644 --- a/core/gallery/importers/importers.go +++ b/core/gallery/importers/importers.go @@ -115,6 +115,7 @@ var defaultImporters = []Importer{ &NemoImporter{}, &FasterWhisperImporter{}, &QwenASRImporter{}, + &FunASRImporter{}, // ParakeetCppImporter matches only parakeet GGUFs (--.gguf); // kept ahead of LlamaCPPImporter so its .gguf bundles aren't claimed by the // generic GGUF importer. diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go index 7e34b7b3ed47..65b0b7322327 100644 --- a/core/gallery/importers/importers_test.go +++ b/core/gallery/importers/importers_test.go @@ -252,7 +252,7 @@ var _ = Describe("DiscoverModelConfig", func() { for _, imp := range registry { names = append(names, imp.Name()) } - Expect(names).To(ContainElements("llama-cpp", "mlx", "vllm", "transformers", "diffusers", "longcat-video")) + Expect(names).To(ContainElements("funasr", "llama-cpp", "mlx", "vllm", "transformers", "diffusers", "longcat-video")) }) It("LlamaCPPImporter exposes name/modality/autodetect", func() { diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 996732fb84d3..a3e63caa7167 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -87,7 +87,7 @@ var _ = Describe("Backend Endpoints", func() { names = append(names, b.Name) } Expect(names).To(ContainElements( - "llama-cpp", "mlx", "vllm", "transformers", "diffusers", + "funasr", "llama-cpp", "mlx", "vllm", "transformers", "diffusers", "privacy-filter", )) @@ -176,7 +176,7 @@ var _ = Describe("Backend Endpoints", func() { } // Importer registry entries that auto-detect. - for _, n := range []string{"llama-cpp", "mlx", "vllm", "transformers", "diffusers"} { + for _, n := range []string{"funasr", "llama-cpp", "mlx", "vllm", "transformers", "diffusers"} { entry, ok := byName[n] Expect(ok).To(BeTrue(), "%s must be present", n) Expect(entry.AutoDetect).To(BeTrue(), "%s must have AutoDetect=true", n) diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 43a5efc6e06b..62ae59fe7cb0 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -125,7 +125,7 @@ For getting started, see the available backends in LocalAI here: https://github. LocalAI supports various types of backends: - **LLM Backends**: For running language models (e.g., llama.cpp, vLLM, SGLang, transformers, MLX) -- **Speech-to-Text Backends**: For transcription and speaker diarization (e.g., whisper.cpp, parakeet.cpp, moss-transcribe.cpp, faster-whisper, NeMo) +- **Speech-to-Text Backends**: For transcription and speaker diarization (e.g., whisper.cpp, parakeet.cpp, moss-transcribe.cpp, faster-whisper, FunASR/SenseVoice, NeMo) - **Text-to-Speech Backends**: For speech synthesis (e.g., piper, Kokoro, VibeVoice, Qwen3-TTS) - **Sound Generation Backends**: For music and audio generation (e.g., ACE-Step) - **Sound Classification Backends**: For sound-event classification / audio tagging - identifying everyday sounds like baby cry, glass breaking, alarms (e.g., ced.cpp)